{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:11:41Z","timestamp":1755839501612,"version":"3.40.3"},"publisher-location":"Cham","reference-count":47,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031282430"},{"type":"electronic","value":"9783031282447"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-28244-7_20","type":"book-chapter","created":{"date-parts":[[2023,3,16]],"date-time":"2023-03-16T17:03:18Z","timestamp":1678986198000},"page":"313-329","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Bootstrapped nDCG Estimation in\u00a0the\u00a0Presence of\u00a0Unjudged Documents"],"prefix":"10.1007","author":[{"given":"Maik","family":"Fr\u00f6be","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Lukas","family":"Gienapp","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Martin","family":"Potthast","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Matthias","family":"Hagen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,3,17]]},"reference":[{"key":"20_CR1","doi-asserted-by":"crossref","unstructured":"Aslam, J.A., Pavlu, V., Yilmaz, E.: A statistical method for system evaluation using incomplete judgments. In: Efthimiadis, E.N., Dumais, S.T., Hawking, D., J\u00e4rvelin, K. (eds.) SIGIR 2006: Proceedings of the 29th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, Seattle, Washington, USA, 6\u201311 August 2006, pp. 541\u2013548. ACM (2006)","DOI":"10.1145\/1148170.1148263"},{"key":"20_CR2","doi-asserted-by":"crossref","unstructured":"Aslam, J.A., Yilmaz, E.: Inferring document relevance from incomplete information. In: Silva, M.J., Laender, A.H.F., Baeza-Yates, R.A., McGuinness, D.L., Olstad, B., Olsen, \u00d8.H., Falc\u00e3o, A.O. (eds.) Proceedings of the Sixteenth ACM Conference on Information and Knowledge Management, CIKM 2007, Lisbon, Portugal, 6\u201310 November 2007, pp. 633\u2013642. ACM (2007)","DOI":"10.1145\/1321440.1321529"},{"key":"20_CR3","unstructured":"Bernstein, Y., Zobel, J.: Redundant documents and search effectiveness. In: Herzog, O., Schek, H., Fuhr, N., Chowdhury, A., Teiken, W. (eds.) Proceedings of the 2005 ACM CIKM International Conference on Information and Knowledge Management, Bremen, Germany, October 31\u2013November 5, 2005, pp. 736\u2013743. ACM (2005)"},{"key":"20_CR4","doi-asserted-by":"crossref","unstructured":"Buckley, C., Voorhees, E.M.: Retrieval evaluation with incomplete information. In: Sanderson, M., J\u00e4rvelin, K., Allan, J., Bruza, P. (eds.) SIGIR 2004: Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, Sheffield, UK, 25\u201329 July 2004, pp. 25\u201332. ACM (2004)","DOI":"10.1145\/1008992.1009000"},{"key":"20_CR5","doi-asserted-by":"crossref","unstructured":"B\u00fcttcher, S., Clarke, C.L.A., Yeung, P.C.K., Soboroff, I.: Reliable information retrieval evaluation with incomplete and biased judgements. In: Kraaij, W., de Vries, A.P., Clarke, C.L.A., Fuhr, N., Kando, N. (eds.) SIGIR 2007: Proceedings of the 30th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, Amsterdam, The Netherlands, 23\u201327 July 2007, pp. 63\u201370. ACM (2007)","DOI":"10.1145\/1277741.1277755"},{"key":"20_CR6","doi-asserted-by":"crossref","unstructured":"Carterette, B., Allan, J., Sitaraman, R.K.: Minimal test collections for retrieval evaluation. In: Efthimiadis, E.N., Dumais, S.T., Hawking, D., J\u00e4rvelin, K. (eds.) SIGIR 2006: Proceedings of the 29th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, Seattle, Washington, USA, 6\u201311 August 2006, pp. 268\u2013275. ACM (2006)","DOI":"10.1145\/1148170.1148219"},{"key":"20_CR7","unstructured":"Carterette, B., Jones, R.: Evaluating search engines by modeling the relationship between relevance and clicks. In: Platt, J.C., Koller, D., Singer, Y., Roweis, S.T. (eds.) Advances in Neural Information Processing Systems 20, Proceedings of the Twenty-First Annual Conference on Neural Information Processing Systems, Vancouver, British Columbia, Canada, 3\u20136 December 2007, pp. 217\u2013224. Curran Associates, Inc. (2007)"},{"key":"20_CR8","doi-asserted-by":"crossref","unstructured":"Clarke, C.L.A., Craswell, N., Soboroff, I.: Overview of the TREC 2009 Web track. In: Voorhees, E.M., Buckland, L.P. (eds.) Proceedings of The Eighteenth Text REtrieval Conference, TREC 2009, Gaithersburg, Maryland, USA, 17\u201320 November 2009, NIST Special Publication, vol. 500\u2013278. National Institute of Standards and Technology (NIST) (2009)","DOI":"10.6028\/NIST.SP.500-278.web-overview"},{"key":"20_CR9","doi-asserted-by":"crossref","unstructured":"Clarke, C.L.A., Craswell, N., Soboroff, I., Cormack, G.V.: Overview of the TREC 2010 Web track. In: Proceedings of The Nineteenth Text REtrieval Conference, TREC 2010, Gaithersburg, Maryland, USA, 16\u201319 November 2010 (2010)","DOI":"10.6028\/NIST.SP.500-294.web-overview"},{"key":"20_CR10","doi-asserted-by":"crossref","unstructured":"Clarke, C.L.A., Craswell, N., Soboroff, I., Voorhees, E.M.: Overview of the TREC 2011 web track. In: Proceedings of The Twentieth Text REtrieval Conference, TREC 2011, Gaithersburg, Maryland, USA, 15\u201318 November 2011 (2011)","DOI":"10.6028\/NIST.SP.500-296.web-overview"},{"key":"20_CR11","doi-asserted-by":"crossref","unstructured":"Clarke, C.L.A., Craswell, N., Voorhees, E.M.: Overview of the TREC 2012 Web track. In: Voorhees, E.M., Buckland, L.P. (eds.) Proceedings of The Twenty-First Text REtrieval Conference, TREC 2012, Gaithersburg, Maryland, USA, 6\u20139 November 2012, NIST Special Publication, vol. 500\u2013298. National Institute of Standards and Technology (NIST) (2012)","DOI":"10.6028\/NIST.SP.500-298.web-overview"},{"key":"20_CR12","doi-asserted-by":"crossref","unstructured":"Cleverdon, C.: The Cranfield tests on index language devices. In: ASLIB Proceedings, pp. 173\u2013192, MCB UP Ltd. (Reprinted in Readings in Information Retrieval, Karen Sparck-Jones and Peter Willett, editors, Morgan Kaufmann, 1997) (1967)","DOI":"10.1108\/eb050097"},{"key":"20_CR13","doi-asserted-by":"crossref","unstructured":"Cleverdon, C.W.: The significance of the Cranfield tests on index languages. In: Bookstein, A., Chiaramella, Y., Salton, G., Raghavan, V.V. (eds.) Proceedings of the 14th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval. Chicago, Illinois, USA, 13\u201316 October 1991 (Special Issue of the SIGIR Forum), pp. 3\u201312. ACM (1991)","DOI":"10.1145\/122860.122861"},{"key":"20_CR14","doi-asserted-by":"crossref","unstructured":"Collins-Thompson, K., Bennett, P.N., Diaz, F., Clarke, C., Voorhees, E.M.: TREC 2013 Web track overview. In: Proceedings of The Twenty-Second Text REtrieval Conference, TREC 2013, Gaithersburg, Maryland, USA, 19\u201322 November 2013 (2013)","DOI":"10.6028\/NIST.SP.500-302.web-overview"},{"key":"20_CR15","doi-asserted-by":"crossref","unstructured":"Collins-Thompson, K., Macdonald, C., Bennett, P.N., Diaz, F., Voorhees, E.M.: TREC 2014 Web track overview. In: Proceedings of the Twenty-Third Text REtrieval Conference, TREC 2014, Gaithersburg, Maryland, USA, 19\u201321 November 2014 (2014)","DOI":"10.6028\/NIST.SP.500-308.web-overview"},{"key":"20_CR16","doi-asserted-by":"crossref","unstructured":"Cormack, G.V., Lynam, T.R.: Statistical precision of information retrieval evaluation. In: Efthimiadis, E.N., Dumais, S.T., Hawking, D., J\u00e4rvelin, K. (eds.) SIGIR 2006: Proceedings of the 29th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, Seattle, Washington, USA, 6\u201311 August 2006, pp. 533\u2013540. ACM (2006)","DOI":"10.1145\/1148170.1148262"},{"key":"20_CR17","volume-title":"28th International Text Retrieval Conference, TREC 2019","author":"N Craswell","year":"2019","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D., Voorhees, E.M.: Overview of the TREC 2019 Deep Learning track. In: Voorhees, E., Ellis, A. (eds.) 28th International Text Retrieval Conference, TREC 2019. Maryland, USA, NIST Special Publication, National Institute of Standards and Technology (NIST) (Nov, Gaithersburg (2019)"},{"key":"20_CR18","doi-asserted-by":"crossref","unstructured":"Efron, B., Tibshirani, R.: An introduction to the bootstrap. CRC Press (1994)","DOI":"10.1201\/9780429246593"},{"key":"20_CR19","doi-asserted-by":"crossref","unstructured":"Ferro, N., Sanderson, M.: How do you test a test?: A multifaceted examination of significance tests. In: Candan, K.S., Liu, H., Akoglu, L., Dong, X.L., Tang, J. (eds.) WSDM \u201922: The Fifteenth ACM International Conference on Web Search and Data Mining, Virtual Event\/Tempe, AZ, USA, 21\u201325 February, 2022, pp. 280\u2013288. ACM (2022)","DOI":"10.1145\/3488560.3498406"},{"key":"20_CR20","doi-asserted-by":"crossref","unstructured":"Fr\u00f6be, M., Bevendorff, J., Gienapp, L., V\u00f6lske, M., Stein, B., Potthast, M., Hagen, M.: CopyCat: Near-duplicates within and between the ClueWeb and the Common Crawl. In: Diaz, F., Shah, C., Suel, T., Castells, P., Jones, R., Sakai, T. (eds.) 44th International ACM Conference on Research and Development in Information Retrieval (SIGIR 2021), pp. 2398\u20132404. ACM, July 2021","DOI":"10.1145\/3404835.3463246"},{"key":"20_CR21","doi-asserted-by":"crossref","unstructured":"Fr\u00f6be, M., Bevendorff, J., Reimer, J., Potthast, M., Hagen, M.: Sampling bias due to near-duplicates in learning to rank. In: 43rd International ACM Conference on Research and Development in Information Retrieval (SIGIR 2020), pp. 1997\u20132000. ACM, July 2020","DOI":"10.1145\/3397271.3401212"},{"key":"20_CR22","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"12","DOI":"10.1007\/978-3-030-45442-5_2","volume-title":"Advances in Information Retrieval","author":"M Fr\u00f6be","year":"2020","unstructured":"Fr\u00f6be, M., Bittner, J.P., Potthast, M., Hagen, M.: The effect of content-equivalent near-duplicates on the evaluation of search engines. In: Jose, J.M., Yilmaz, E., Magalh\u00e3es, J., Castells, P., Ferro, N., Silva, M.J., Martins, F. (eds.) ECIR 2020. LNCS, vol. 12036, pp. 12\u201319. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-45442-5_2"},{"issue":"4","key":"20_CR23","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin, K., Kek\u00e4l\u00e4inen, J.: Cumulated gain-based evaluation of IR techniques. ACM Trans. Inf. Syst. 20(4), 422\u2013446 (2002)","journal-title":"ACM Trans. Inf. Syst."},{"key":"20_CR24","doi-asserted-by":"crossref","unstructured":"Lin, J., Yang, P.: The impact of score ties on repeatability in document ranking. In: Piwowarski, B., Chevalier, M., Gaussier, \u00c9., Maarek, Y., Nie, J., Scholer, F. (eds.) Proceedings of the 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2019, Paris, France, 21\u201325 July 2019, pp. 1125\u20131128. ACM (2019)","DOI":"10.1145\/3331184.3331339"},{"issue":"4","key":"20_CR25","doi-asserted-by":"publisher","first-page":"416","DOI":"10.1007\/s10791-016-9282-6","volume":"19","author":"X Lu","year":"2016","unstructured":"Lu, X., Moffat, A., Culpepper, J.S.: The effect of pooling and evaluation depth on IR metrics. Inf. Retrieval J. 19(4), 416\u2013445 (2016). https:\/\/doi.org\/10.1007\/s10791-016-9282-6","journal-title":"Inf. Retrieval J."},{"key":"20_CR26","doi-asserted-by":"crossref","unstructured":"Macdonald, C., Tonellotto, N.: Declarative experimentation in information retrieval using PyTerrier. In: Balog, K., Setty, V., Lioma, C., Liu, Y., Zhang, M., Berberich, K. (eds.) ICTIR \u201920: The 2020 ACM SIGIR International Conference on the Theory of Information Retrieval, Virtual Event, Norway, 14\u201317 September 2020, pp. 161\u2013168. ACM (2020)","DOI":"10.1145\/3409256.3409829"},{"key":"20_CR27","doi-asserted-by":"crossref","unstructured":"Moffat, A., Zobel, J.: Rank-biased precision for measurement of retrieval effectiveness. ACM Trans. Inf. Syst. 27(1), 2:1\u20132:27 (2008)","DOI":"10.1145\/1416950.1416952"},{"key":"20_CR28","doi-asserted-by":"crossref","unstructured":"Palotti, J.R.M., Scells, H., Zuccon, G.: TrecTools: an open-source Python library for information retrieval practitioners involved in TREC-like campaigns. In: Piwowarski, B., Chevalier, M., Gaussier, \u00c9., Maarek, Y., Nie, J., Scholer, F. (eds.) Proceedings of the 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2019, Paris, France, 21\u201325 July 2019, pp. 1325\u20131328, ACM (2019)","DOI":"10.1145\/3331184.3331399"},{"issue":"9","key":"20_CR29","doi-asserted-by":"publisher","first-page":"1431","DOI":"10.1093\/jamia\/ocaa091","volume":"27","author":"K Roberts","year":"2020","unstructured":"Roberts, K., Alam, T., Bedrick, S., Demner-Fushman, D., Lo, K., Soboroff, I., Voorhees, E.M., Wang, L.L., Hersh, W.R.: TREC-COVID: rationale and structure of an information retrieval shared task for COVID-19. J. Am. Medical Informatics Assoc. 27(9), 1431\u20131436 (2020)","journal-title":"J. Am. Medical Informatics Assoc."},{"key":"20_CR30","doi-asserted-by":"crossref","unstructured":"Sakai, T.: Evaluating evaluation metrics based on the bootstrap. In: Efthimiadis, E.N., Dumais, S.T., Hawking, D., J\u00e4rvelin, K. (eds.) SIGIR 2006: Proceedings of the 29th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, Seattle, Washington, USA, 6\u201311 August 2006, pp. 525\u2013532. ACM (2006)","DOI":"10.1145\/1148170.1148261"},{"key":"20_CR31","doi-asserted-by":"crossref","unstructured":"Sakai, T.: Alternatives to bpref. In: Kraaij, W., de Vries, A.P., Clarke, C.L.A., Fuhr, N., Kando, N. (eds.) SIGIR 2007: Proceedings of the 30th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, Amsterdam, The Netherlands, 23\u201327 July 2007, pp. 71\u201378. ACM (2007)","DOI":"10.1145\/1277741.1277756"},{"issue":"2","key":"20_CR32","doi-asserted-by":"publisher","first-page":"531","DOI":"10.1016\/j.ipm.2006.07.020","volume":"43","author":"T Sakai","year":"2007","unstructured":"Sakai, T.: On the reliability of information retrieval metrics based on graded relevance. Inf. Process. Manag. 43(2), 531\u2013548 (2007)","journal-title":"Inf. Process. Manag."},{"key":"20_CR33","doi-asserted-by":"crossref","unstructured":"Sakai, T.: Comparing metrics across TREC and NTCIR: The robustness to system bias. In: Shanahan, J.G., Amer-Yahia, S., Manolescu, I., Zhang, Y., Evans, D.A., Kolcz, A., Choi, K., Chowdhury, A. (eds.) Proceedings of the 17th ACM Conference on Information and Knowledge Management, CIKM 2008, Napa Valley, California, USA, October 26\u201330, 2008, pp. 581\u2013590. ACM (2008)","DOI":"10.1145\/1458082.1458159"},{"issue":"4","key":"20_CR34","doi-asserted-by":"publisher","first-page":"495","DOI":"10.1016\/S0306-4573(97)00027-7","volume":"33","author":"J Savoy","year":"1997","unstructured":"Savoy, J.: Statistical inference in retrieval effectiveness evaluation. Inf. Process. Manag. 33(4), 495\u2013512 (1997)","journal-title":"Inf. Process. Manag."},{"key":"20_CR35","doi-asserted-by":"crossref","unstructured":"Smucker, M.D., Allan, J., Carterette, B.: A comparison of statistical significance tests for information retrieval evaluation. In: Silva, M.J., Laender, A.H.F., Baeza-Yates, R.A., McGuinness, D.L., Olstad, B., Olsen, \u00d8.H., Falc\u00e3o, A.O. (eds.) Proceedings of the Sixteenth ACM Conference on Information and Knowledge Management, CIKM 2007, Lisbon, Portugal, 6\u201310 November 2007, pp. 623\u2013632. ACM (2007)","DOI":"10.1145\/1321440.1321528"},{"key":"20_CR36","unstructured":"Thakur, N., Reimers, N., R\u00fcckl\u00e9, A., Srivastava, A., Gurevych, I.: BEIR: A heterogeneous benchmark for zero-shot evaluation of information retrieval models. In: Vanschoren, J., Yeung, S. (eds.) Proceedings of the Neural Information Processing Systems Track on Datasets and Benchmarks 1, NeurIPS Datasets and Benchmarks 2021, December 2021, virtual (2021)"},{"issue":"1","key":"20_CR37","doi-asserted-by":"publisher","first-page":"11","DOI":"10.1145\/1067268.1067272","volume":"39","author":"E Voorhees","year":"2005","unstructured":"Voorhees, E.: The TREC robust retrieval track. SIGIR Forum 39(1), 11\u201320 (2005)","journal-title":"SIGIR Forum"},{"key":"20_CR38","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"355","DOI":"10.1007\/3-540-45691-0_34","volume-title":"Evaluation of Cross-Language Information Retrieval Systems","author":"EM Voorhees","year":"2002","unstructured":"Voorhees, E.M.: The philosophy of information retrieval evaluation. In: Peters, C., Braschler, M., Gonzalo, J., Kluck, M. (eds.) CLEF 2001. LNCS, vol. 2406, pp. 355\u2013370. Springer, Heidelberg (2002). https:\/\/doi.org\/10.1007\/3-540-45691-0_34"},{"key":"20_CR39","doi-asserted-by":"crossref","unstructured":"Voorhees, E.M.: The effect of sampling strategy on inferred measures. In: Geva, S., Trotman, A., Bruza, P., Clarke, C.L.A., J\u00e4rvelin, K. (eds.) The 37th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR \u201914, Gold Coast, QLD, Australia\u2013July 06\u201311, 2014, pp. 1119\u20131122. ACM (2014)","DOI":"10.1145\/2600428.2609524"},{"key":"20_CR40","series-title":"The Information Retrieval Series","doi-asserted-by":"publisher","first-page":"45","DOI":"10.1007\/978-3-030-22948-1_2","volume-title":"Information Retrieval Evaluation in a Changing World","author":"EM Voorhees","year":"2019","unstructured":"Voorhees, E.M.: The evolution of cranfield. In: Information Retrieval Evaluation in a Changing World. TIRS, vol. 41, pp. 45\u201369. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-22948-1_2"},{"key":"20_CR41","doi-asserted-by":"crossref","unstructured":"Voorhees, E.M., et al.: TREC-COVID: constructing a pandemic information retrieval test collection. SIGIR Forum 54(1), 1:1\u20131:12 (2020)","DOI":"10.1145\/3451964.3451965"},{"key":"20_CR42","unstructured":"Voorhees, E.M., Soboroff, I., Lin, J.: Can old TREC collections reliably evaluate modern neural retrieval models? CoRR abs\/2201.11086 (2022)"},{"key":"20_CR43","unstructured":"Xiong, L., Xiong, C., Li, Y., Tang, K., Liu, J., Bennett, P.N., Ahmed, J., Overwijk, A.: Approximate nearest neighbor negative contrastive learning for dense text retrieval. In: 9th International Conference on Learning Representations, ICLR 2021, Virtual Event, Austria, 3\u20137 May 2021, OpenReview.net (2021)"},{"key":"20_CR44","doi-asserted-by":"crossref","unstructured":"Yilmaz, E., Aslam, J.A.: Estimating average precision with incomplete and imperfect judgments. In: Yu, P.S., Tsotras, V.J., Fox, E.A., Liu, B. (eds.) Proceedings of the 2006 ACM CIKM International Conference on Information and Knowledge Management, Arlington, Virginia, USA, 6\u201311 November 2006, pp. 102\u2013111. ACM (2006)","DOI":"10.1145\/1183614.1183633"},{"key":"20_CR45","doi-asserted-by":"crossref","unstructured":"Yilmaz, E., Kanoulas, E., Aslam, J.A.: A simple and efficient sampling method for estimating AP and NDCG. In: Myaeng, S., Oard, D.W., Sebastiani, F., Chua, T., Leong, M. (eds.) Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2008, Singapore, 20\u201324 July 2008, pp. 603\u2013610. ACM (2008)","DOI":"10.1145\/1390334.1390437"},{"key":"20_CR46","doi-asserted-by":"crossref","unstructured":"Zobel, J.: How reliable are the results of large-scale information retrieval experiments? In: Croft, W.B., Moffat, A., van Rijsbergen, C.J., Wilkinson, R., Zobel, J. (eds.) SIGIR \u201998: Proceedings of the 21st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, 24\u201328 August 1998, Melbourne, Australia, pp. 307\u2013314. ACM (1998)","DOI":"10.1145\/290941.291014"},{"key":"20_CR47","doi-asserted-by":"crossref","unstructured":"Zobel, J., Rashidi, L.: Corpus bootstrapping for assessment of the properties of effectiveness measures. In: d\u2019Aquin, M., Dietze, S., Hauff, C., Curry, E., Cudr\u00e9-Mauroux, P. (eds.) CIKM \u201920: The 29th ACM International Conference on Information and Knowledge Management, Virtual Event, Ireland, 19\u201323 October 2020, pp. 1933\u20131952. ACM (2020)","DOI":"10.1145\/3340531.3411998"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-28244-7_20","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,16]],"date-time":"2024-10-16T15:09:03Z","timestamp":1729091343000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-28244-7_20"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031282430","9783031282447"],"references-count":47,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-28244-7_20","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"17 March 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Dublin","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Ireland","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2 April 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"6 April 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"45","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2023.org\/index.html?v=1.0","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"489","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"77","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"83","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"16% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}