{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,11,11]],"date-time":"2026-11-11T07:41:36Z","timestamp":1794382896618,"version":"4.9.0"},"publisher-location":"Berlin, Heidelberg","reference-count":23,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"value":"9783642171864","type":"print"},{"value":"9783642171871","type":"electronic"}],"license":[{"start":{"date-parts":[[2010,1,1]],"date-time":"2010-01-01T00:00:00Z","timestamp":1262304000000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2010]]},"DOI":"10.1007\/978-3-642-17187-1_9","type":"book-chapter","created":{"date-parts":[[2010,12,3]],"date-time":"2010-12-03T15:33:24Z","timestamp":1291390404000},"page":"97-109","source":"Crossref","is-referenced-by-count":4,"title":["Score Estimation, Incomplete Judgments, and Significance Testing in IR Evaluation"],"prefix":"10.1007","author":[{"given":"Sri Devi","family":"Ravana","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alistair","family":"Moffat","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","reference":[{"key":"9_CR1","doi-asserted-by":"crossref","unstructured":"Aslam, J., Yilmaz, E.: Inferring document relevance from incomplete information. In: Proc. 2007 ACM CIKM Conf. Lisbon, Portugal, pp. 603\u2013610 (November 2007)","DOI":"10.1145\/1321440.1321529"},{"key":"9_CR2","doi-asserted-by":"crossref","unstructured":"Aslam, J.A., Pavlu, V., Yilmaz, E.: A statistical method for system evaluation using incomplete judgments. In: Proc. 29th ACM SIGIR Conf. Seattle, WA, pp. 541\u2013548 (August 2006)","DOI":"10.1145\/1148170.1148263"},{"key":"9_CR3","doi-asserted-by":"crossref","unstructured":"Bompada, T., Chang, C.C., Chen, J., Kumar, R., Shenoy, R.: On the robustness of relevance measures with incomplete judgments. In: Proc. 30th ACM SIGIR Conf. Amsterdam, pp. 359\u2013366 (July 2007)","DOI":"10.1145\/1277741.1277804"},{"key":"9_CR4","doi-asserted-by":"crossref","unstructured":"Buckley, C., Voorhees, E.M.: Evaluating evaluation measure stability. In: Proc. 23rd ACM SIGIR Conf. Athens, Greece, pp. 33\u201340 (July 2000)","DOI":"10.1145\/345508.345543"},{"key":"9_CR5","doi-asserted-by":"crossref","unstructured":"Buckley, C., Voorhees, E.M.: Retrieval evaluation with incomplete information. In: Proc. 27th ACM SIGIR Conf. Sheffield, England, pp. 25\u201332 (July 2004)","DOI":"10.1145\/1008992.1009000"},{"key":"9_CR6","doi-asserted-by":"crossref","unstructured":"B\u00fcttcher, S., Clarke, C.L.A., Yeung, P.C.K., Soboroff, I.: Reliable information retrieval evaluation with incomplete and biased judgements. In: Proc. 30th ACM SIGIR Conf. pp. 63\u201370 (July 2007)","DOI":"10.1145\/1277741.1277755"},{"key":"9_CR7","doi-asserted-by":"crossref","unstructured":"Carterette, B., Smucker, M.D.: Hypothesis testing with incomplete relevance judgments. In: Proc. 2007 ACM CIKM Conf, Lisbon, Portugal, pp. 643\u2013652 (November 2007)","DOI":"10.1145\/1321440.1321530"},{"key":"9_CR8","doi-asserted-by":"crossref","unstructured":"Cormack, G.V., Lynam, T.R.: Validity and power of t-test for comparing MAP and GMAP. In: Proc. 30th ACM SIGIR Conf. pp. 753\u2013754 (July 2007)","DOI":"10.1145\/1277741.1277892"},{"key":"9_CR9","doi-asserted-by":"crossref","unstructured":"Hawking, D.: Overview of the TREC-9 Web Track. In: Proc. 9th Text REtrieval Conf. (TREC-9). Gaithersburg, Maryland (November 2000)","DOI":"10.6028\/NIST.SP.500-249.web-overview"},{"key":"9_CR10","doi-asserted-by":"crossref","unstructured":"Huffman, S.B., Hochster, M.: How well does result relevance predict session satisfaction? In: Proc. 30th ACM SIGIR Conf. Amsterdam, pp. 567\u2013574 (July 2007)","DOI":"10.1145\/1277741.1277839"},{"issue":"4","key":"9_CR11","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K. J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin, K., Kek\u00e4l\u00e4inen, J.: Cumulated gain-based evaluation of IR techniques. ACM Transactions on Information Systems\u00a020(4), 422\u2013446 (2002)","journal-title":"ACM Transactions on Information Systems"},{"issue":"1","key":"9_CR12","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/1416950.1416952","volume":"27","author":"A. Moffat","year":"2008","unstructured":"Moffat, A., Zobel, J.: Rank-biased precision for measurement of retrieval effectiveness. ACM Transactions on Information Systems\u00a027(1), 1\u201327 (2008)","journal-title":"ACM Transactions on Information Systems"},{"key":"9_CR13","doi-asserted-by":"crossref","unstructured":"Sakai, T.: Evaluating evaluation metrics based on the bootstrap. In: Proc. 29th ACM SIGIR Conf. Seattle, WA, pp. 525\u2013534 (August 2006)","DOI":"10.1145\/1148170.1148261"},{"key":"9_CR14","doi-asserted-by":"crossref","unstructured":"Sakai, T.: Alternatives to Bpref. In: Proc. 30th ACM SIGIR Conf, Amsterdam, pp. 71\u201378 (July 2007)","DOI":"10.1145\/1277741.1277756"},{"issue":"5","key":"9_CR15","doi-asserted-by":"publisher","first-page":"447","DOI":"10.1007\/s10791-008-9059-7","volume":"11","author":"T. Sakai","year":"2008","unstructured":"Sakai, T., Kando, N.: On information retrieval metrics designed for evaluation with incomplete relevance assessments. Information Retrieval\u00a011(5), 447\u2013470 (2008)","journal-title":"Information Retrieval"},{"key":"9_CR16","doi-asserted-by":"crossref","unstructured":"Sanderson, M., Zobel, J.: Information retrieval system evaluation: Effort, sensitivity, and reliability. In: Proc. 28th ACM SIGIR Conf. Salvador, Brazil, pp. 162\u2013169 (August 2005)","DOI":"10.1145\/1076034.1076064"},{"key":"9_CR17","doi-asserted-by":"crossref","unstructured":"Smucker, M.D., Allan, J., Carterette, B.: A comparison of statistical significance tests for information retrieval. In: Proc. 2007 ACM CIKM Conf, Lisbon, pp. 623\u2013632 (November 2007)","DOI":"10.1145\/1321440.1321528"},{"key":"9_CR18","doi-asserted-by":"crossref","unstructured":"Smucker, M.D., Allan, J., Carterette, B.: Agreement among statistical significance tests for information retrieval evaluation at varying sample sizes. In: Proc. 32nd ACM SIGIR Conf. Boston, MA, pp. 630\u2013631 (July 2009)","DOI":"10.1145\/1571941.1572050"},{"key":"9_CR19","doi-asserted-by":"crossref","unstructured":"Turpin, A., Scholer, F.: User performance versus precision measures for simple search tasks. In: Proc. 29th ACM SIGIR Conf. pp. 11\u201318 (August 2006)","DOI":"10.1145\/1148170.1148176"},{"key":"9_CR20","volume-title":"TREC: Experiment and Evaluation in Information Retrieval","author":"E.M. Voorhees","year":"2005","unstructured":"Voorhees, E.M., Harman, D.K.: TREC: Experiment and Evaluation in Information Retrieval. The MIT Press, Cambridge (2005)"},{"key":"9_CR21","doi-asserted-by":"crossref","unstructured":"Webber, W., Park, L.A.F.: Score adjustment for correction of pooling bias. In: Proc. 32nd ACM SIGIR Conf. Boston, MA, pp. 444\u2013451 (July 2009)","DOI":"10.1145\/1571941.1572018"},{"key":"9_CR22","doi-asserted-by":"crossref","unstructured":"Yilmaz, E., Kanoulas, E., Aslam, J.A.: A simple and efficient sampling method for estimating AP and NDCG. In: Proc. 31st ACM SIGIR Conf. Singapore, pp. 603\u2013610 (July 2008)","DOI":"10.1145\/1390334.1390437"},{"key":"9_CR23","doi-asserted-by":"crossref","unstructured":"Zobel, J.: How reliable are the results of large-scale information retrieval experiments? In: Proc. 21st ACM SIGIR Conf. Melbourne, Australia, pp. 307\u2013314 (August 1998)","DOI":"10.1145\/290941.291014"}],"container-title":["Lecture Notes in Computer Science","Information Retrieval Technology"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-642-17187-1_9","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,2,28]],"date-time":"2025-02-28T07:13:54Z","timestamp":1740726834000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-642-17187-1_9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2010]]},"ISBN":["9783642171864","9783642171871"],"references-count":23,"URL":"https:\/\/doi.org\/10.1007\/978-3-642-17187-1_9","relation":{"is-part-of":[{"id-type":"doi","id":"https:\/\/doi.org\/10.1007\/978-3-642-17187-1","asserted-by":"object"}]},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2010]]}}}