{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,19]],"date-time":"2026-03-19T14:45:31Z","timestamp":1773931531485,"version":"3.50.1"},"publisher-location":"Cham","reference-count":53,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030157111","type":"print"},{"value":"9783030157128","type":"electronic"}],"license":[{"start":{"date-parts":[[2019,1,1]],"date-time":"2019-01-01T00:00:00Z","timestamp":1546300800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2019]]},"DOI":"10.1007\/978-3-030-15712-8_41","type":"book-chapter","created":{"date-parts":[[2019,4,7]],"date-time":"2019-04-07T03:27:59Z","timestamp":1554607679000},"page":"636-651","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":6,"title":["Correlation, Prediction and Ranking of Evaluation Metrics in Information Retrieval"],"prefix":"10.1007","author":[{"given":"Soumyajit","family":"Gupta","sequence":"first","affiliation":[]},{"given":"Mucahid","family":"Kutlu","sequence":"additional","affiliation":[]},{"given":"Vivek","family":"Khetan","sequence":"additional","affiliation":[]},{"given":"Matthew","family":"Lease","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,4,7]]},"reference":[{"key":"41_CR1","unstructured":"Alonso, O., Mizzaro, S.: Can we get rid of TREC assessors? using mechanical turk for relevance assessment. In: Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation, vol. 15, p. 16 (2009)"},{"key":"41_CR2","doi-asserted-by":"crossref","unstructured":"Armstrong, T.G., Moffat, A., Webber, W., Zobel, J.: Improvements that don\u2019t add up: ad-hoc retrieval results since 1998. In: Proceedings of the 18th ACM Conference on Information and Knowledge Management, pp. 601\u2013610. ACM (2009)","DOI":"10.1145\/1645953.1646031"},{"key":"41_CR3","doi-asserted-by":"crossref","unstructured":"Aslam, J.A., Pavlu, V., Yilmaz, E.: A statistical method for system evaluation using incomplete judgments. In: Proceedings of the 29th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 541\u2013548. ACM (2006)","DOI":"10.1145\/1148170.1148263"},{"key":"41_CR4","doi-asserted-by":"crossref","unstructured":"Aslam, J.A., Yilmaz, E.: Inferring document relevance from incomplete information. In: Proceedings of the Sixteenth ACM Conference on Information and Knowledge Management, pp. 633\u2013642. ACM (2007)","DOI":"10.1145\/1321440.1321529"},{"key":"41_CR5","doi-asserted-by":"crossref","unstructured":"Aslam, J.A., Yilmaz, E., Pavlu, V.: A geometric interpretation of r-precision and its correlation with average precision. In: Proceedings of the 28th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 573\u2013574. ACM (2005)","DOI":"10.1145\/1076034.1076134"},{"key":"41_CR6","doi-asserted-by":"crossref","unstructured":"Aslam, J.A., Yilmaz, E., Pavlu, V.: The maximum entropy method for analyzing retrieval measures. In: Proceedings of the 28th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 27\u201334. ACM (2005)","DOI":"10.1145\/1076034.1076042"},{"issue":"3","key":"41_CR7","doi-asserted-by":"publisher","first-page":"693","DOI":"10.1007\/s10115-011-0391-7","volume":"30","author":"A Baccini","year":"2012","unstructured":"Baccini, A., D\u00e9jean, S., Lafage, L., Mothe, J.: How many performance measures to evaluate information retrieval systems? Knowl. Inf. Syst. 30(3), 693 (2012)","journal-title":"Knowl. Inf. Syst."},{"key":"41_CR8","unstructured":"de Bruijn, L., Martin, J.: Literature mining in molecular biology. In: Proceedings of the EFMI Workshop on Natural Language Processing in Biomedical Applications, pp. 1\u20135 (2002)"},{"key":"41_CR9","doi-asserted-by":"crossref","unstructured":"Buckley, C., Voorhees, E.M.: Retrieval evaluation with incomplete information. In: Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 25\u201332. ACM (2004)","DOI":"10.1145\/1008992.1009000"},{"key":"41_CR10","unstructured":"Buckley, C., Voorhees, E.M.: Retrieval system evaluation. In: TREC: Experiment and Evaluation in Information Retrieval, pp. 53\u201375 (2005)"},{"key":"41_CR11","doi-asserted-by":"crossref","unstructured":"Carterette, B., Allan, J., Sitaraman, R.: Minimal test collections for retrieval evaluation. In: Proceedings of the 29th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 268\u2013275. ACM (2006)","DOI":"10.1145\/1148170.1148219"},{"key":"41_CR12","doi-asserted-by":"crossref","unstructured":"Chapelle, O., Metlzer, D., Zhang, Y., Grinspan, P.: Expected reciprocal rank for graded relevance. In: Proceedings of the 18th ACM Conference on Information and Knowledge Management, pp. 621\u2013630. ACM (2009)","DOI":"10.1145\/1645953.1646033"},{"key":"41_CR13","doi-asserted-by":"crossref","unstructured":"Clarke, C., Craswell, N.: Overview of the TREC 2011 web track. In: TREC (2011)","DOI":"10.6028\/NIST.SP.500-296.web-overview"},{"key":"41_CR14","doi-asserted-by":"crossref","unstructured":"Clarke, C., Craswell, N., Soboroff, I., Cormack, G.: Overview of the TREC 2010 web track. In: TREC (2010)","DOI":"10.6028\/NIST.SP.500-294.web-overview"},{"key":"41_CR15","doi-asserted-by":"crossref","unstructured":"Clarke, C., Craswell, N., Voorhees, E.M.: Overview of the TREC 2012 web track. In: TREC (2012)","DOI":"10.6028\/NIST.SP.500-298.web-overview"},{"key":"41_CR16","doi-asserted-by":"crossref","unstructured":"Collins-Thompson, K., Bennett, P., Clarke, C., Voorhees, E.M.: TREC 2013 web track overview. In: TREC (2013)","DOI":"10.6028\/NIST.SP.500-302.web-overview"},{"key":"41_CR17","doi-asserted-by":"crossref","unstructured":"Collins-Thompson, K., Macdonald, C., Bennett, P., Voorhees, E.M.: TREC 2014 web track overview. In: TREC (2014)","DOI":"10.6028\/NIST.SP.500-308.web-overview"},{"key":"41_CR18","doi-asserted-by":"crossref","unstructured":"Cormack, G.V., Palmer, C.R., Clarke, C.L.: Efficient construction of large test collections. In: Proceedings of the 21st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 282\u2013289. ACM (1998)","DOI":"10.1145\/290941.291009"},{"issue":"2","key":"41_CR19","doi-asserted-by":"publisher","first-page":"856","DOI":"10.1016\/j.ipm.2007.03.014","volume":"44","author":"L Egghe","year":"2008","unstructured":"Egghe, L.: The measures precision, recall, fallout and miss as a function of the number of retrieved documents and their mutual interrelations. Inf. Process. Manage. 44(2), 856\u2013876 (2008). Evaluating Exploratory Search Systems Digital Libraries in the Context of Users Broader Activities","journal-title":"Inf. Process. Manage."},{"key":"41_CR20","unstructured":"Grady, C., Lease, M.: Crowdsourcing document relevance assessment with mechanical turk. In: Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk, pp. 172\u2013179. Association for Computational Linguistics (2010)"},{"issue":"4","key":"41_CR21","doi-asserted-by":"publisher","first-page":"21","DOI":"10.1145\/1629096.1629099","volume":"27","author":"J Guiver","year":"2009","unstructured":"Guiver, J., Mizzaro, S., Robertson, S.: A few good topics: experiments in topic set reduction for retrieval evaluation. ACM Trans. Inf. Syst. (TOIS) 27(4), 21 (2009)","journal-title":"ACM Trans. Inf. Syst. (TOIS)"},{"key":"41_CR22","doi-asserted-by":"crossref","unstructured":"Hawking, D.: Overview of the TREC-9 web track. In: TREC (2000)","DOI":"10.1145\/344250.344254"},{"issue":"12","key":"41_CR23","doi-asserted-by":"publisher","first-page":"1553","DOI":"10.1093\/bioinformatics\/18.12.1553","volume":"18","author":"L Hirschman","year":"2002","unstructured":"Hirschman, L., Park, J.C., Tsujii, J., Wong, L., Wu, C.H.: Accomplishments and challenges in literature data mining for biology. Bioinformatics 18(12), 1553\u20131561 (2002)","journal-title":"Bioinformatics"},{"key":"41_CR24","doi-asserted-by":"crossref","unstructured":"Hosseini, M., Cox, I.J., Milic-Frayling, N., Shokouhi, M., Yilmaz, E.: An uncertainty-aware query selection model for evaluation of IR systems. In: Proceedings of the 35th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 901\u2013910. ACM (2012)","DOI":"10.1145\/2348283.2348403"},{"key":"41_CR25","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"113","DOI":"10.1007\/978-3-642-23318-0_12","volume-title":"Advances in Information Retrieval Theory","author":"M Hosseini","year":"2011","unstructured":"Hosseini, M., Cox, I.J., Milic-Frayling, N., Vinay, V., Sweeting, T.: Selecting a subset of queries for acquisition of further relevance judgements. In: Amati, G., Crestani, F. (eds.) ICTIR 2011. LNCS, vol. 6931, pp. 113\u2013124. Springer, Heidelberg (2011). https:\/\/doi.org\/10.1007\/978-3-642-23318-0_12"},{"key":"41_CR26","unstructured":"Ishioka, T.: Evaluation of criteria for information retrieval. In: IEEE\/WIC International Conference on Web Intelligence, 2003. WI 2003. Proceedings, pp. 425\u2013431. IEEE (2003)"},{"key":"41_CR27","unstructured":"Jones, K.S., van Rijsbergen, C.J.: Report on the need for and provision of an \"ideal\" information retrieval test collection (British library research and development report no. 5266), p. 43 (1975)"},{"key":"41_CR28","doi-asserted-by":"crossref","unstructured":"Jones, T., Thomas, P., Scholer, F., Sanderson, M.: Features of disagreement between retrieval effectiveness measures. In: Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 847\u2013850. ACM (2015)","DOI":"10.1145\/2766462.2767824"},{"issue":"4","key":"41_CR29","doi-asserted-by":"publisher","first-page":"416","DOI":"10.1007\/s10791-016-9282-6","volume":"19","author":"X Lu","year":"2016","unstructured":"Lu, X., Moffat, A., Culpepper, J.S.: The effect of pooling and evaluation depth on IR metrics. Inf. Retrieval J. 19(4), 416\u2013445 (2016)","journal-title":"Inf. Retrieval J."},{"key":"41_CR30","doi-asserted-by":"crossref","unstructured":"Mizzaro, S., Robertson, S.: Hits hits TREC: exploring IR evaluation results with network analysis. In: Proceedings of the 30th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 479\u2013486. ACM (2007)","DOI":"10.1145\/1277741.1277824"},{"key":"41_CR31","doi-asserted-by":"crossref","unstructured":"Moffat, A., Webber, W., Zobel, J.: Strategic system comparisons via targeted relevance judgments. In: Proceedings of the 30th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 375\u2013382. ACM (2007)","DOI":"10.1145\/1277741.1277806"},{"issue":"1","key":"41_CR32","doi-asserted-by":"publisher","first-page":"2","DOI":"10.1145\/1416950.1416952","volume":"27","author":"A Moffat","year":"2008","unstructured":"Moffat, A., Zobel, J.: Rank-biased precision for measurement of retrieval effectiveness. ACM Trans. Inf. Syst. (TOIS) 27(1), 2 (2008)","journal-title":"ACM Trans. Inf. Syst. (TOIS)"},{"issue":"2","key":"41_CR33","doi-asserted-by":"publisher","first-page":"301","DOI":"10.1016\/j.joi.2012.12.001","volume":"7","author":"SI Moghadasi","year":"2013","unstructured":"Moghadasi, S.I., Ravana, S.D., Raman, S.N.: Low-cost evaluation techniques for information retrieval systems: a review. J. Informetr. 7(2), 301\u2013312 (2013)","journal-title":"J. Informetr."},{"issue":"3","key":"41_CR34","doi-asserted-by":"publisher","first-page":"595","DOI":"10.1016\/j.ipm.2005.03.023","volume":"42","author":"R Nuray","year":"2006","unstructured":"Nuray, R., Can, F.: Automatic ranking of information retrieval systems using data fusion. Inf. Process. Manage. 42(3), 595\u2013614 (2006)","journal-title":"Inf. Process. Manage."},{"key":"41_CR35","first-page":"96","volume":"15","author":"CH Papadimitriou","year":"1984","unstructured":"Papadimitriou, C.H.: The largest subdeterminant of a matrix. Bull. Math. Soc. Greece 15, 96\u2013105 (1984)","journal-title":"Bull. Math. Soc. Greece"},{"key":"41_CR36","unstructured":"Park, L., Zhang, Y.: On the distribution of user persistence for rank-biased precision. In: Proceedings of the 12th Australasian Document Computing Symposium, pp. 17\u201324 (2007)"},{"key":"41_CR37","unstructured":"Pavlu, V., Aslam, J.: A practical sampling strategy for efficient retrieval evaluation. Technical report, College of Computer and Information Science, Northeastern University (2007)"},{"issue":"5","key":"41_CR38","doi-asserted-by":"publisher","first-page":"447","DOI":"10.1007\/s10791-008-9059-7","volume":"11","author":"T Sakai","year":"2008","unstructured":"Sakai, T., Kando, N.: On information retrieval metrics designed for evaluation with incomplete relevance assessments. Inf. Retrieval 11(5), 447\u2013470 (2008)","journal-title":"Inf. Retrieval"},{"key":"41_CR39","doi-asserted-by":"crossref","unstructured":"Sakai, T.: Alternatives to Bpref. In: Proceedings of the 30th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 71\u201378. ACM (2007)","DOI":"10.1145\/1277741.1277756"},{"issue":"4","key":"41_CR40","first-page":"1163","volume":"2","author":"T Sakai","year":"2007","unstructured":"Sakai, T.: On the properties of evaluation metrics for finding one highly relevant document. Inf. Media Technol. 2(4), 1163\u20131180 (2007)","journal-title":"Inf. Media Technol."},{"issue":"2","key":"41_CR41","doi-asserted-by":"publisher","first-page":"531","DOI":"10.1016\/j.ipm.2006.07.020","volume":"43","author":"T Sakai","year":"2007","unstructured":"Sakai, T.: On the reliability of information retrieval metrics based on graded relevance. Inf. Process. Manage. 43(2), 531\u2013548 (2007)","journal-title":"Inf. Process. Manage."},{"key":"41_CR42","doi-asserted-by":"crossref","unstructured":"Sanderson, M.: Test Collection Based Evaluation of Information Retrieval Systems. Now Publishers Inc (2010)","DOI":"10.1561\/1500000009"},{"key":"41_CR43","first-page":"681","volume":"51","author":"C Sheffield","year":"1985","unstructured":"Sheffield, C.: Selecting band combinations from multispectral data. Photogramm. Eng. Remote Sens. 51, 681\u2013687 (1985)","journal-title":"Photogramm. Eng. Remote Sens."},{"key":"41_CR44","doi-asserted-by":"crossref","unstructured":"Soboroff, I., Nicholas, C., Cahan, P.: Ranking retrieval systems without relevance judgments. In: Proceedings of the 24th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 66\u201373. ACM (2001)","DOI":"10.1145\/383952.383961"},{"key":"41_CR45","unstructured":"Tague-Sutcliffe, J., Blustein, J.: Overview of TREC 2001. In: Proceedings of the Third Text Retrieval Conference (TREC-3), pp. 385\u2013398 (1995)"},{"key":"41_CR46","unstructured":"Thom, J., Scholer, F.: A comparison of evaluation measures given how users perform on search tasks. In: ADCS2007 Australasian Document Computing Symposium. RMIT University, School of Computer Science and Information Technology (2007)"},{"issue":"5","key":"41_CR47","doi-asserted-by":"publisher","first-page":"697","DOI":"10.1016\/S0306-4573(00)00010-8","volume":"36","author":"EM Voorhees","year":"2000","unstructured":"Voorhees, E.M.: Variations in relevance judgments and the measurement of retrieval effectiveness. Inf. Process. Manage. 36(5), 697\u2013716 (2000)","journal-title":"Inf. Process. Manage."},{"key":"41_CR48","doi-asserted-by":"crossref","unstructured":"Voorhees, E.M.: Overview of the TREC 2004 robust track. In: TREC, vol. 4 (2004)","DOI":"10.6028\/NIST.SP.500-261.robust-overview"},{"key":"41_CR49","doi-asserted-by":"crossref","unstructured":"Voorhees, E.M., Harman, D.: Overview of TREC 2001. In: TREC (2001)","DOI":"10.6028\/NIST.SP.500-250.overview-overview"},{"key":"41_CR50","doi-asserted-by":"crossref","unstructured":"Voorhees, E.M., Tice, D.M.: The TREC-8 question answering track evaluation. In: TREC, vol. 1999, p. 82 (1999)","DOI":"10.6028\/NIST.SP.500-246.qa-overview"},{"key":"41_CR51","doi-asserted-by":"crossref","unstructured":"Webber, W., Moffat, A., Zobel, J., Sakai, T.: Precision-at-ten considered redundant. In: Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 695\u2013696. ACM (2008)","DOI":"10.1145\/1390334.1390456"},{"key":"41_CR52","doi-asserted-by":"crossref","unstructured":"Yilmaz, E., Aslam, J.A.: Estimating average precision with incomplete and imperfect judgments. In: Proceedings of the 15th ACM International Conference on Information and Knowledge Management, pp. 102\u2013111. ACM (2006)","DOI":"10.1145\/1183614.1183633"},{"issue":"2","key":"41_CR53","doi-asserted-by":"publisher","first-page":"173","DOI":"10.1007\/s10115-007-0101-7","volume":"16","author":"E Yilmaz","year":"2008","unstructured":"Yilmaz, E., Aslam, J.A.: Estimating average precision when judgments are incomplete. Knowl. Inf. Syst. 16(2), 173\u2013211 (2008)","journal-title":"Knowl. Inf. Syst."}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-15712-8_41","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,16]],"date-time":"2024-07-16T17:53:52Z","timestamp":1721152432000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-15712-8_41"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019]]},"ISBN":["9783030157111","9783030157128"],"references-count":53,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-15712-8_41","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2019]]},"assertion":[{"value":"7 April 2019","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Cologne","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Germany","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2019","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14 April 2019","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18 April 2019","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"41","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2019","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2019.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}