{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:01:34Z","timestamp":1784178094320,"version":"3.55.0"},"reference-count":38,"publisher":"Springer Science and Business Media LLC","issue":"5","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Inf Retrieval"],"published-print":{"date-parts":[[2008,10]]},"DOI":"10.1007\/s10791-008-9059-7","type":"journal-article","created":{"date-parts":[[2008,3,19]],"date-time":"2008-03-19T16:12:05Z","timestamp":1205943125000},"page":"447-470","source":"Crossref","is-referenced-by-count":97,"title":["On information retrieval metrics designed for evaluation with incomplete relevance assessments"],"prefix":"10.1007","volume":"11","author":[{"given":"Tetsuya","family":"Sakai","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Noriko","family":"Kando","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2008,3,20]]},"reference":[{"key":"9059_CR1","doi-asserted-by":"crossref","unstructured":"Aslam, J. A., & Savell, R. (2003). On the effectiveness of evaluating retrieval systems in the absence of relevance judgments. In ACM SIGIR 2003 Proceedings (pp. 361\u2013362).","DOI":"10.1145\/860435.860501"},{"key":"9059_CR2","doi-asserted-by":"crossref","unstructured":"Aslam, J. A., Pavlu, V., & Yilmaz, E. (2006). A statistical method for system evaluation using incomplete judgments. In ACM SIGIR 2006 Proceedings (pp. 541\u2013548).","DOI":"10.1145\/1148170.1148263"},{"key":"9059_CR3","doi-asserted-by":"crossref","unstructured":"Bompada, T., Chang, C.-C., Chen, J., Kumar, R., & Shenoy, R. (2007). On the robustness of relevance measures with incomplete judgments. In ACM SIGIR 2007 Proceedings (pp. 359\u2013366).","DOI":"10.1145\/1277741.1277804"},{"key":"9059_CR4","doi-asserted-by":"crossref","unstructured":"Buckley, C., & Voorhees, E. M. (2004). Retrieval evaluation with incomplete information. In ACM SIGIR 2004 Proceedings (pp. 25\u201332).","DOI":"10.1145\/1008992.1009000"},{"key":"9059_CR5","doi-asserted-by":"crossref","unstructured":"Burges, C., Shaked, T., Renshaw, E., Lazier, A., Deeds, M., Hamilton, N., & Hullender, G. (2005). Learning to rank using gradient descent. In ACM ICML 2005 Proceedings (pp. 89\u201396).","DOI":"10.1145\/1102351.1102363"},{"key":"9059_CR6","doi-asserted-by":"crossref","unstructured":"B\u00fcttcher, S., Clarke, C. L. A., Yeung, P. C. K., & Soboroff, I. (2007). Reliable information retrieval evaluation with incomplete and biased judgements. In ACM SIGIR 2007 Proceedings (pp. 63\u201370).","DOI":"10.1145\/1277741.1277755"},{"key":"9059_CR7","doi-asserted-by":"crossref","unstructured":"Carterette, B., Allan, J., & Sitaraman, R. (2006). Minimal test collections for retrieval evaluation. In ACM SIGIR 2006 Proceedings (pp. 268\u2013275).","DOI":"10.1145\/1148170.1148219"},{"key":"9059_CR8","doi-asserted-by":"crossref","unstructured":"Cleverdon, C. W. (1967). The cranfield tests on index language devices. In Aslib Proceedings (Vol. 19, pp. 173\u2013192).","DOI":"10.1108\/eb050097"},{"key":"9059_CR9","doi-asserted-by":"crossref","unstructured":"Cormack, G. V., Palmer, C. R., & Clarke, C. L. A. (1998). Efficient construction of large test collections. In ACM SIGIR \u201998 Proceedings (pp. 282\u2013289).","DOI":"10.1145\/290941.291009"},{"key":"9059_CR10","unstructured":"De Beer, J., & Moens, M.-F. (2006). Rpref\u2014a generalization of bpref towards graded relevance judgments. In ACM SIGIR 2006 Proceedings (pp. 637\u2013638)."},{"key":"9059_CR11","doi-asserted-by":"crossref","unstructured":"Efron, B., & Tibshirani, R. (1993). Introduction to the bootstrap. Chapman & Hall\/CRC.","DOI":"10.1007\/978-1-4899-4541-9"},{"key":"9059_CR12","unstructured":"Gr\u00f6nqvist, L. (2005). Evaluating latent semantic vector models with synonym tests and document retrieval. In ELECTRA Workshop\u2014Methodologies and Evaluating of Lexical Cohesion Techniques in Real-World Applications (pp. 86\u201388)."},{"issue":"4","key":"9059_CR13","doi-asserted-by":"crossref","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K. J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin, K., & Kek\u00e4l\u00e4inen, J. (2002). Cumulated gain-based evaluation of IR techniques. ACM Transactions on Information Systems, 20(4), 422\u2013446.","journal-title":"ACM Transactions on Information Systems"},{"key":"9059_CR14","unstructured":"Kando, N. (2007). Overview of the Sixth NTCIR Workshop. In NTCIR-6 Proceedings (pp. i\u2013ix)."},{"key":"9059_CR15","doi-asserted-by":"crossref","first-page":"1019","DOI":"10.1016\/j.ipm.2005.01.004","volume":"41","author":"J. Kek\u00e4l\u00e4inen","year":"2005","unstructured":"Kek\u00e4l\u00e4inen, J. (2005). Binary and graded relevance in IR evaluations\u2014comparison of the effects on ranking of IR systems. Information Processing and Management, 41, 1019\u20131033.","journal-title":"Information Processing and Management"},{"key":"9059_CR17","doi-asserted-by":"crossref","unstructured":"Moffat, A., Webber, W., & Zobel, J. (2007). Strategic system comparisons via targeted relevance judgments. In ACM SIGIR 2007 Proceedings (pp. 375\u2013382).","DOI":"10.1145\/1277741.1277806"},{"key":"9059_CR16","doi-asserted-by":"crossref","unstructured":"Moffat, A., & Zobel, J. (2008). Rank-biased precision for measurement of retrieval effectiveness (under review).","DOI":"10.1145\/1416950.1416952"},{"key":"9059_CR18","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2006a). Bootstrap-based comparisons of IR metrics for finding one relevant document. In AIRS 2006: Lecture Notes in Computer Science 4182 (pp. 374\u2013389). Springer-Verlag.","DOI":"10.1007\/11880592_29"},{"key":"9059_CR19","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2006b). Evaluating evaluation metrics based on the bootstrap. In ACM SIGIR 2006 Proceedings (pp. 525\u2013532).","DOI":"10.1145\/1148170.1148261"},{"key":"9059_CR20","unstructured":"Sakai, T. (2006c). On the task of finding one highly relevant document with high precision. Information Processing Society of Japan Transactions on Databases, 47(SIG4(TOD29)), 13\u201327"},{"key":"9059_CR21","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2007a). Alternatives to Bpref. In ACM SIGIR 2007 Proceedings (pp. 71\u201378).","DOI":"10.1145\/1277741.1277756"},{"key":"9059_CR22","unstructured":"Sakai, T. (2007b). Evaluating information retrieval metrics based on bootstrap hypothesis tests. Information Processing Society of Japan Transactions on Databases 48(SIG9(TOD35)), 11\u201328"},{"key":"9059_CR23","unstructured":"Sakai, T. (2007c). evaluating the task of finding one relevant document using incomplete relevance data. In Forum on Information Technology 2007 Information Technology Letters (pp. 91\u201394)."},{"key":"9059_CR24","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2007d). On penalising late arrival of relevant documents in information retrieval evaluation with graded relevance. In Proceedings of the First International Workshop on Evaluating Information Acess (EVIA 2007) (pp. 32\u201343).","DOI":"10.1016\/j.ipm.2006.07.020"},{"issue":"SIG9(TOD35)","key":"9059_CR25","first-page":"29","volume":"48","author":"T. Sakai","year":"2007","unstructured":"Sakai, T. (2007e). On the properties of evaluation metrics for finding one highly relevant document. Information Processing Society of Japan Transactions on Databases, 48(SIG9(TOD35)), 29\u201346.","journal-title":"Information Processing Society of Japan Transactions on Databases"},{"issue":"2","key":"9059_CR26","doi-asserted-by":"crossref","first-page":"531","DOI":"10.1016\/j.ipm.2006.07.020","volume":"43","author":"T. Sakai","year":"2007","unstructured":"Sakai, T. (2007f). On the reliability of information retrieval metrics based on graded relevance. Information Processing and Management, 43(2), 531\u2013548.","journal-title":"Information Processing and Management"},{"key":"9059_CR27","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2008a). Comparing metrics across TREC and NTCIR: The robustness to pool depth bias (under review).","DOI":"10.1145\/1390334.1390454"},{"key":"9059_CR28","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2008b). Comparing metrics across TREC and NTCIR: The robustness to system bias (under review).","DOI":"10.1145\/1458082.1458159"},{"key":"9059_CR29","doi-asserted-by":"crossref","unstructured":"Sakai, T., & Sparck Jones, K. (2001). Generic summaries for indexing in information retrieval. In ACM SIGIR 2001 Proceedings (pp. 190\u2013198).","DOI":"10.1145\/383952.383987"},{"key":"9059_CR30","doi-asserted-by":"crossref","unstructured":"Soboroff, I., Nicholas, C., & Cahan, P. (2001). Ranking retrieval systems without relevance judgments. In ACM SIGIR 2001 Proceedings (pp. 66\u201373).","DOI":"10.1145\/383952.383961"},{"key":"9059_CR31","unstructured":"Sormunen, E. (2002). Liberal relevance criteria of TREC\u2014counting on negligible documents? In ACM SIGIR 2002 Proceedings (pp. 324\u2013330)."},{"key":"9059_CR32","doi-asserted-by":"crossref","unstructured":"Voorhees, E. M. (2001). Evaluation by highly relevant documents. In ACM SIGIR 2001 Proceedings (pp. 74\u201382).","DOI":"10.1145\/383952.383963"},{"key":"9059_CR33","doi-asserted-by":"crossref","unstructured":"Voorhees, E. M. (2002). The effect of topic set size on retrieval experiment error. In ACM SIGIR 2002 Proceedings (pp. 316\u2013323).","DOI":"10.1145\/564376.564432"},{"key":"9059_CR34","unstructured":"Voorhees, E. M. (2004). Overview of the TREC 2003 robust retrieval Track. In TREC 2003 Proceedings."},{"key":"9059_CR35","unstructured":"Voorhees, E. M. (2005). Overview of the TREC 2004 robust retrieval track. In TREC 2004 proceedings."},{"key":"9059_CR36","doi-asserted-by":"crossref","unstructured":"Voorhees, E. M., & Buckley, C. (2002). The philosophy of information retrieval evaluation. In Proceedings of CLEF 2001, Lecture Notes in Computer Science 2406 (pp. 355\u2013370).","DOI":"10.1007\/3-540-45691-0_34"},{"key":"9059_CR37","doi-asserted-by":"crossref","unstructured":"Yilmaz, E., & Aslam, J. A. (2006). Estimating average precision with incomplete and imperfect judgments. In ACM CIKM 2006 Proceedings (pp. 102\u2013111).","DOI":"10.1145\/1183614.1183633"},{"key":"9059_CR38","doi-asserted-by":"crossref","unstructured":"Zobel, J. (1998). How Reliable are the results of large-scale information retrieval experiments? In ACM SIGIR \u201998 Proceedings (pp. 307\u2013314).","DOI":"10.1145\/290941.291014"}],"container-title":["Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-008-9059-7.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,5,3]],"date-time":"2020-05-03T07:26:17Z","timestamp":1588490777000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10791-008-9059-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2008,3,20]]},"references-count":38,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2008,10]]}},"alternative-id":["9059"],"URL":"https:\/\/doi.org\/10.1007\/s10791-008-9059-7","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"value":"1386-4564","type":"print"},{"value":"1573-7659","type":"electronic"}],"subject":[],"published":{"date-parts":[[2008,3,20]]}}}