{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,4,24]],"date-time":"2025-04-24T04:08:54Z","timestamp":1745467734983,"version":"3.40.4"},"publisher-location":"Berlin, Heidelberg","reference-count":22,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"type":"print","value":"9783642353406"},{"type":"electronic","value":"9783642353413"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2012]]},"DOI":"10.1007\/978-3-642-35341-3_3","type":"book-chapter","created":{"date-parts":[[2012,12,5]],"date-time":"2012-12-05T17:57:39Z","timestamp":1354730259000},"page":"26-38","source":"Crossref","is-referenced-by-count":5,"title":["The Reusability of a Diversified Search Test Collection"],"prefix":"10.1007","author":[{"given":"Tetsuya","family":"Sakai","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhicheng","family":"Dou","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ruihua","family":"Song","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Noriko","family":"Kando","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","reference":[{"issue":"6","key":"3_CR1","doi-asserted-by":"publisher","first-page":"491","DOI":"10.1007\/s10791-007-9032-x","volume":"10","author":"C. Buckley","year":"2007","unstructured":"Buckley, C., Dimmick, D., Soboroff, I., Voorhees, E.M.: Bias and the limits of pooling for large collections. Information Retrieval\u00a010(6), 491\u2013508 (2007)","journal-title":"Information Retrieval"},{"key":"3_CR2","doi-asserted-by":"crossref","unstructured":"Buckley, C., Voorhees, E.M.: Retrieval evaluation with incomplete information. In: Proceedings of ACM SIGIR 2004, pp. 25\u201332 (2004)","DOI":"10.1145\/1008992.1009000"},{"key":"3_CR3","doi-asserted-by":"crossref","unstructured":"B\u00fcttcher, S., Clarke, C.L., Yeung, P.C., Soboroff, I.: Reliable information retrieval evaluation with incomplete and biased judgements. In: ACM SIGIR 2007 Proceedings, pp. 63\u201370 (2007)","DOI":"10.1145\/1277741.1277755"},{"key":"3_CR4","doi-asserted-by":"crossref","unstructured":"Carterette, B.: Multiple testing in statistical analysis of systems-based information retrieval experiments. ACM TOIS\u00a030(1) (2012)","DOI":"10.1145\/2094072.2094076"},{"key":"3_CR5","doi-asserted-by":"crossref","unstructured":"Carterette, B., Gabrilovich, E., Josifovski, V., Metzler, D.: Measuring the reusability of test collections. In: Proceedings of WSDM 2012, pp. 231\u2013240 (2010)","DOI":"10.1145\/1718487.1718516"},{"key":"3_CR6","doi-asserted-by":"crossref","unstructured":"Carterette, B., Kanoulas, E., Pavlu, V., Fang, H.: Reusable test collections through experimental design. In: Proceedings of ACM SIGIR 2010, pp. 547\u2013554 (2010)","DOI":"10.1145\/1835449.1835541"},{"key":"3_CR7","doi-asserted-by":"crossref","unstructured":"Carterette, B., Pavlu, V., Kanoulas, E., Aslam, J.A., Allan, J.: Evaluation over thousands of queries. In: Proceedings of ACM SIGIR 2008, pp. 651\u2013658 (2008)","DOI":"10.1145\/1390334.1390445"},{"key":"3_CR8","doi-asserted-by":"crossref","unstructured":"Clarke, C.L., Craswell, N., Soboroff, I.: Overview of the TREC 2009 web track. In: Proceedings of TREC 2009 (2009)","DOI":"10.6028\/NIST.SP.500-278.web-overview"},{"key":"3_CR9","doi-asserted-by":"crossref","unstructured":"Clarke, C.L., Craswell, N., Soboroff, I., Voorhees, E.: Overview of the TREC 2011 web track. In: Proceedings of TREC 2011 (2012)","DOI":"10.6028\/NIST.SP.500-296.web-overview"},{"key":"3_CR10","doi-asserted-by":"crossref","unstructured":"Sakai, T.: Alternatives to bpref. In: Proceedings of ACM SIGIR 2007, pp. 71\u201378 (2007)","DOI":"10.1145\/1277741.1277756"},{"key":"3_CR11","doi-asserted-by":"crossref","unstructured":"Sakai, T.: Comparing metrics across TREC and NTCIR: The robustness to pool depth bias. In: Proceedings of ACM SIGIR 2008, pp. 691\u2013692 (2008)","DOI":"10.1145\/1390334.1390454"},{"key":"3_CR12","doi-asserted-by":"crossref","unstructured":"Sakai, T.: Comparing metrics across TREC and NTCIR: The robustness to system bias. In: Proceedings of ACM CIKM 2008, pp. 581\u2013590 (2008)","DOI":"10.1145\/1458082.1458159"},{"key":"3_CR13","unstructured":"Sakai, T., Kando, N.: Are popular documents more likely to be relevant? a dive into the ACLIA IR4QA pools. In: Proceedings of EVIA 2008, pp. 8\u20139 (2008)"},{"key":"3_CR14","doi-asserted-by":"publisher","first-page":"447","DOI":"10.1007\/s10791-008-9059-7","volume":"11","author":"T. Sakai","year":"2008","unstructured":"Sakai, T., Kando, N.: On information retrieval metrics designed for evaluation with incomplete relevance assessments. Information Retrieval\u00a011, 447\u2013470 (2008)","journal-title":"Information Retrieval"},{"key":"3_CR15","unstructured":"Sakai, T., Song, R.: Diversified search evaluation: Lessons from the NTCIR-9 INTENT task. Information Retrieval (to appear)"},{"key":"3_CR16","doi-asserted-by":"crossref","unstructured":"Sakai, T., Song, R.: Evaluating diversified search results using per-intent graded relevance. In: Proceedings of ACM SIGIR 2011 (2011)","DOI":"10.1145\/2009916.2010055"},{"key":"3_CR17","unstructured":"Song, R., Zhang, M., Sakai, T., Kato, M.P., Liu, Y., Sugimoto, M., Wang, Q., Orii, N.: Overview of the NTCIR-9 INTENT task. In: Proceedings of NTCIR-9, pp. 82\u2013105 (2011)"},{"key":"3_CR18","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"355","DOI":"10.1007\/3-540-45691-0_34","volume-title":"Evaluation of Cross-Language Information Retrieval Systems","author":"E.M. Voorhees","year":"2002","unstructured":"Voorhees, E.M.: The Philosophy of Information Retrieval Evaluation. In: Peters, C., Braschler, M., Gonzalo, J., Kluck, M. (eds.) CLEF 2001. LNCS, vol.\u00a02406, pp. 355\u2013370. Springer, Heidelberg (2002)"},{"key":"3_CR19","doi-asserted-by":"crossref","unstructured":"Webber, W., Park, L.A.: Score adjustment for correction of pooling bias. In: Proceedings of ACM SIGIR 2009, pp. 444\u2013451 (2009)","DOI":"10.1145\/1571941.1572018"},{"key":"3_CR20","doi-asserted-by":"crossref","unstructured":"Yilmaz, E., Aslam, J., Robertson, S.: A new rank correlation coefficient for information retrieval. In: Proceedings of ACM SIGIR 2008, pp. 587\u2013594 (2008)","DOI":"10.1145\/1390334.1390435"},{"key":"3_CR21","doi-asserted-by":"crossref","unstructured":"Yilmaz, E., Aslam, J.A.: Estimating average precision with incomplete and imperfect judgments. In: ACM CIKM 2006 Proceedings, pp. 102\u2013111 (2006)","DOI":"10.1145\/1183614.1183633"},{"key":"3_CR22","doi-asserted-by":"crossref","unstructured":"Zobel, J.: How reliable are the results of large-scale information retrieval experiments? In: Proceedings of ACM SIGIR 1998, pp. 307\u2013314 (1998)","DOI":"10.1145\/290941.291014"}],"container-title":["Lecture Notes in Computer Science","Information Retrieval Technology"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-642-35341-3_3.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,23]],"date-time":"2025-04-23T08:21:27Z","timestamp":1745396487000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-642-35341-3_3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012]]},"ISBN":["9783642353406","9783642353413"],"references-count":22,"URL":"https:\/\/doi.org\/10.1007\/978-3-642-35341-3_3","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2012]]}}}