{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,16]],"date-time":"2026-01-16T20:38:11Z","timestamp":1768595891878,"version":"3.49.0"},"reference-count":51,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2015,12,23]],"date-time":"2015-12-23T00:00:00Z","timestamp":1450828800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2015,12,23]],"date-time":"2015-12-23T00:00:00Z","timestamp":1450828800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval J"],"published-print":{"date-parts":[[2016,6]]},"DOI":"10.1007\/s10791-015-9274-y","type":"journal-article","created":{"date-parts":[[2015,12,23]],"date-time":"2015-12-23T11:17:23Z","timestamp":1450869443000},"page":"313-350","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":20,"title":["Test collection reliability: a study of bias and robustness to statistical assumptions via stochastic simulation"],"prefix":"10.1007","volume":"19","author":[{"given":"Juli\u00e1n","family":"Urbano","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2015,12,23]]},"reference":[{"key":"9274_CR1","unstructured":"Allen, M. J., & Yen, W. M. (1979). Introduction to measurement theory. California: Wadsworth."},{"key":"9274_CR2","doi-asserted-by":"crossref","unstructured":"Bailey, P., Craswell, N., Soboroff, I., Thomas, P., de\u00a0Vries, A. P., & Yilmaz, E. (2008). Relevance assessment: Are judges exchangeable and does it matter? In International ACM SIGIR conference on research and development in information retrieval, pp. 667\u2013674.","DOI":"10.1145\/1390334.1390447"},{"key":"9274_CR3","doi-asserted-by":"crossref","unstructured":"Bodoff, D., & Li, P. (2007). Test theory for assessing IR test collections. In International ACM SIGIR conference on research and development in information retrieval, pp. 367\u2013374.","DOI":"10.1145\/1277741.1277805"},{"key":"9274_CR4","doi-asserted-by":"crossref","unstructured":"Boytsov, L., Belova, A., & Westfall, P. (2013). Deciding on an adjustment for multiplicity in IR experiments. In International ACM SIGIR conference on research and development in information retrieval, pp. 403\u2013412.","DOI":"10.1145\/2484028.2484034"},{"key":"9274_CR5","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4757-3456-0","volume-title":"Generalizability theory","author":"RL Brennan","year":"2001","unstructured":"Brennan, R. L. (2001). Generalizability theory. Berlin: Springer."},{"issue":"3","key":"9274_CR6","doi-asserted-by":"publisher","first-page":"277","DOI":"10.1111\/j.1745-3984.1977.tb00045.x","volume":"14","author":"RL Brennan","year":"1977","unstructured":"Brennan, R. L., & Kane, M. T. (1977). An index of dependability for mastery tests. Journal of Educational Measurement, 14(3), 277\u2013289.","journal-title":"Journal of Educational Measurement"},{"key":"9274_CR7","doi-asserted-by":"crossref","unstructured":"Buckley, C., & Voorhees, E. M. (2000). Evaluating evaluation measure stability. In International ACM SIGIR conference on research and development in information retrieval, pp. 33\u201334.","DOI":"10.1145\/345508.345543"},{"issue":"6","key":"9274_CR8","doi-asserted-by":"publisher","first-page":"491","DOI":"10.1007\/s10791-007-9032-x","volume":"10","author":"C Buckley","year":"2007","unstructured":"Buckley, C., Dimmick, D., Soboroff, I., & Voorhees, E. M. (2007). Bias and the limits of pooling for large collections. Journal of Information Retrieval, 10(6), 491\u2013508.","journal-title":"Journal of Information Retrieval"},{"key":"9274_CR9","doi-asserted-by":"crossref","unstructured":"Carterette, B. (2009). On rank correlation and the distance between rankings. In International ACM SIGIR conference on research and development in information retrieval, pp. 436\u2013443.","DOI":"10.1145\/1571941.1572017"},{"key":"9274_CR10","doi-asserted-by":"publisher","unstructured":"Carterette, B. (2012). Multiple testing in statistical analysis of systems-based information retrieval experiments. ACM Transactions on Information Systems, 30(1), 4. doi:10.1145\/2094072.2094076.","DOI":"10.1145\/2094072.2094076"},{"key":"9274_CR11","doi-asserted-by":"crossref","unstructured":"Carterette, B., & Soboroff, I. (2010). The effect of assessor error on IR system evaluation. In International ACM SIGIR conference on research and development in information retrieval, pp. 539\u2013546.","DOI":"10.1145\/1835449.1835540"},{"key":"9274_CR12","doi-asserted-by":"crossref","unstructured":"Carterette, B., Pavlu, V., Kanoulas, E., Aslam, J. A., & Allan, J. (2009). If I had a million queries. In European conference on information retrieval, pp. 288\u2013300.","DOI":"10.1007\/978-3-642-00958-7_27"},{"key":"9274_CR13","doi-asserted-by":"crossref","unstructured":"Carterette, B., Kanoulas, E., & Yilmaz, E. (2011). Simulating simple user behavior for system effectiveness evaluation. In ACM international conference on information and knowledge management, pp. 611\u2013620.","DOI":"10.1145\/2063576.2063668"},{"key":"9274_CR14","unstructured":"Cohen, J. (1988). Statistical power analysis for the behavioral sciences. New Jersey: Lawrence Erlbaum Associates."},{"key":"9274_CR15","doi-asserted-by":"crossref","unstructured":"Cormack, G. V., & Lynam, T. R. (2006). Statistical precision of information retrieval evaluation. In International ACM SIGIR conference on research and development in information retrieval, pp. 533\u2013540.","DOI":"10.1145\/1148170.1148262"},{"issue":"4","key":"9274_CR16","doi-asserted-by":"publisher","first-page":"907","DOI":"10.1214\/aoms\/1177728067","volume":"27","author":"J Cornfield","year":"1956","unstructured":"Cornfield, J., & Tukey, J. W. (1956). Average values of mean squares in factorials. The Annals of Mathematical Statistics, 27(4), 907\u2013949.","journal-title":"The Annals of Mathematical Statistics"},{"issue":"1","key":"9274_CR17","doi-asserted-by":"publisher","first-page":"141","DOI":"10.1080\/03461238.1928.10416872","volume":"11","author":"H Cram\u00e9r","year":"1928","unstructured":"Cram\u00e9r, H. (1928). On the composition of elementary errors II. Scandinavian Actuarial Journal, 11(1), 141\u2013180.","journal-title":"Scandinavian Actuarial Journal"},{"key":"9274_CR18","volume-title":"The dependability of behavioral measurements: Theory of generalizability for scores and profiles","author":"LJ Cronbach","year":"1972","unstructured":"Cronbach, L. J., Gleser, G. C., Nanda, H., & Rajaratnam, N. (1972). The dependability of behavioral measurements: Theory of generalizability for scores and profiles. London: Wiley."},{"key":"9274_CR19","doi-asserted-by":"crossref","unstructured":"Hull, D. (1993). Using statistical testing in the evaluation of retrieval experiments. In International ACM SIGIR conference on research and development in information retrieval, pp. 329\u2013338.","DOI":"10.1145\/160688.160758"},{"key":"9274_CR20","doi-asserted-by":"crossref","DOI":"10.1201\/b17116","volume-title":"Dependence modeling with copulas","author":"H Joe","year":"2014","unstructured":"Joe, H. (2014). Dependence modeling with copulas. Boca Raton: CRC Press."},{"issue":"4","key":"9274_CR38","doi-asserted-by":"publisher","first-page":"393","DOI":"10.1108\/eb026588","volume":"30","author":"KS Jones","year":"1974","unstructured":"Jones, K. S. (1974). Automatic indexing. Journal of Documentation, 30(4), 393\u2013432. doi:10.1108\/eb026524.","journal-title":"Journal of Documentation"},{"issue":"5","key":"9274_CR21","doi-asserted-by":"publisher","first-page":"1019","DOI":"10.1016\/j.ipm.2005.01.004","volume":"41","author":"J Kek\u00e4l\u00e4inen","year":"2005","unstructured":"Kek\u00e4l\u00e4inen, J. (2005). Binary and graded relevance in IR evaluations: Comparison of the effects on ranking of IR systems. Information Processing and Management, 41(5), 1019\u20131033.","journal-title":"Information Processing and Management"},{"key":"9274_CR22","doi-asserted-by":"crossref","unstructured":"Lin, W. H., & Hauptmann, A. (2005). Revisiting the effect of topic set size on retrieval error. In International ACM SIGIR conference on research and development in information retrieval, pp. 637\u2013638.","DOI":"10.1145\/1076034.1076166"},{"issue":"1","key":"9274_CR23","doi-asserted-by":"publisher","first-page":"18","DOI":"10.1145\/1273221.1273223","volume":"41","author":"M Melucci","year":"2007","unstructured":"Melucci, M. (2007). On rank correlation in information retrieval evaluation. ACM SIGIR Forum, 41(1), 18\u201333.","journal-title":"ACM SIGIR Forum"},{"key":"9274_CR26","doi-asserted-by":"crossref","unstructured":"Robertson, S., & Kanoulas, E. (2012). On per-topic variance in IR evaluation. In International ACM SIGIR conference on research and development in information retrieval, pp. 891\u2013900.","DOI":"10.1145\/2348283.2348402"},{"key":"9274_CR27","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2006). Evaluating evaluation metrics based on the bootstrap. In International ACM SIGIR conference on research and development in information retrieval, pp. 525\u2013532.","DOI":"10.1145\/1148170.1148261"},{"issue":"2","key":"9274_CR28","doi-asserted-by":"publisher","first-page":"531","DOI":"10.1016\/j.ipm.2006.07.020","volume":"43","author":"T Sakai","year":"2007","unstructured":"Sakai, T. (2007). On the reliability of information retrieval metrics based on graded relevance. Information Processing and Management, 43(2), 531\u2013548.","journal-title":"Information Processing and Management"},{"key":"9274_CR29","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2014a). Designing test collections for comparing many systems. In ACM international conference on information and knowledge management, pp. 61\u201370.","DOI":"10.1145\/2661829.2661893"},{"key":"9274_CR30","unstructured":"Sakai, T. (2014b). Topic set size design with variance estimates from two-way ANOVA. In International workshop on evaluating information access, pp. 1\u20138."},{"key":"9274_CR200","doi-asserted-by":"publisher","unstructured":"Sakai, T. (2015). Topic set size design. Journal of Information Retrieval. doi:10.1007\/s10791-015-9273-z.","DOI":"10.1007\/s10791-015-9273-z"},{"issue":"5","key":"9274_CR31","doi-asserted-by":"publisher","first-page":"447","DOI":"10.1007\/s10791-008-9059-7","volume":"11","author":"T Sakai","year":"2008","unstructured":"Sakai, T., & Kando, N. (2008). On information retrieval metrics designed for evaluation with incomplete relevance assessments. Journal of Information Retrieval, 11(5), 447\u2013470.","journal-title":"Journal of Information Retrieval"},{"issue":"4","key":"9274_CR32","doi-asserted-by":"publisher","first-page":"247","DOI":"10.1561\/1500000009","volume":"4","author":"M Sanderson","year":"2010","unstructured":"Sanderson, M. (2010). Test collection based evaluation of information retrieval systems. Foundations and Trends in Information Retrieval, 4(4), 247\u2013375.","journal-title":"Foundations and Trends in Information Retrieval"},{"key":"9274_CR33","doi-asserted-by":"crossref","unstructured":"Sanderson, M., & Zobel, J. (2005). Information retrieval system evaluation: Effort, sensitivity, and reliability. In International ACM SIGIR conference on research and development in information retrieval, pp. 162\u2013169.","DOI":"10.1145\/1076034.1076064"},{"key":"9274_CR34","doi-asserted-by":"crossref","unstructured":"Sanderson, M., Turpin, A., Zhang, Y., & Scholer, F. (2012). Differences in effectiveness across sub-collections. In ACM international conference on information and knowledge management, pp. 1965\u20131969.","DOI":"10.1145\/2396761.2398553"},{"key":"9274_CR35","volume-title":"Variance components","author":"SR Searle","year":"2006","unstructured":"Searle, S. R., Casella, G., & McCulloch, C. E. (2006). Variance components. London: Wiley."},{"key":"9274_CR36","doi-asserted-by":"crossref","unstructured":"Smucker, M. D., Allan, J., & Carterette, B. (2007). A comparison of statistical significance tests for information retrieval evaluation. In ACM international conference on information and knowledge management, pp. 623\u2013632.","DOI":"10.1145\/1321440.1321528"},{"key":"9274_CR37","doi-asserted-by":"crossref","unstructured":"Smucker, M. D., Allan, J., & Carterette, B. (2009). Agreement among statistical significance tests for information retrieval evaluation at varying sample sizes. In International ACM SIGIR conference on research and development in information retrieval, pp. 630\u2013631.","DOI":"10.1145\/1571941.1572050"},{"issue":"4","key":"9274_CR39","doi-asserted-by":"publisher","first-page":"467","DOI":"10.1016\/0306-4573(92)90005-K","volume":"28","author":"J Tague-Sutcliffe","year":"1992","unstructured":"Tague-Sutcliffe, J. (1992). The pragmatics of information retrieval experimentation, revisited. Information Processing and Management, 28(4), 467\u2013490.","journal-title":"Information Processing and Management"},{"key":"9274_CR40","doi-asserted-by":"crossref","unstructured":"Urbano, J., & Marrero, M. (2015). How do gain and discount functions affect the correlation between DCG and user satisfaction? In European conference on information retrieval, pp. 197\u2013202.","DOI":"10.1007\/978-3-319-16354-3_20"},{"key":"9274_CR41","doi-asserted-by":"crossref","unstructured":"Urbano, J., Marrero, M., & Mart\u00edn, D. (2013a). A comparison of the optimality of statistical significance tests for information retrieval evaluation. In International ACM SIGIR conference on research and development in information retrieval, pp. 925\u2013928.","DOI":"10.1145\/2484028.2484163"},{"key":"9274_CR42","doi-asserted-by":"crossref","unstructured":"Urbano, J., Marrero, M., & Mart\u00edn, D. (2013b). On the measurement of test collection reliability. In International ACM SIGIR conference on research and development in information retrieval, pp. 393\u2013402.","DOI":"10.1145\/2484028.2484038"},{"key":"9274_CR25","unstructured":"van Rijsbergen, C. J. (1979). Information retrieval. London: Butterworths."},{"key":"9274_CR24","unstructured":"von Mises, R. (1931). Wahrscheinlichkeitsrechnung und ihre Anwendungen in der Statistik und theoretischen Physik."},{"key":"9274_CR43","doi-asserted-by":"crossref","unstructured":"Voorhees, E. M. (1998). Variations in relevance judgments and the measurement of retrieval effectiveness. In International ACM SIGIR conference on research and development in information retrieval, pp. 315\u2013323.","DOI":"10.1145\/290941.291017"},{"key":"9274_CR44","doi-asserted-by":"crossref","unstructured":"Voorhees, E. M. (2001). Evaluation by highly relevant documents. In International ACM SIGIR conference on research and development in information retrieval, pp. 74\u201382.","DOI":"10.1145\/383952.383963"},{"key":"9274_CR45","doi-asserted-by":"crossref","unstructured":"Voorhees, E. M. (2009). Topic set size redux. In International ACM SIGIR conference on research and development in information retrieval, pp. 806\u2013807.","DOI":"10.1145\/1571941.1572138"},{"key":"9274_CR46","doi-asserted-by":"crossref","unstructured":"Voorhees, E. M., & Buckley, C. (2002). The effect of topic set size on retrieval experiment error. In International ACM SIGIR conference on research and development in information retrieval, pp. 316\u2013323.","DOI":"10.1145\/564376.564432"},{"key":"9274_CR47","doi-asserted-by":"publisher","first-page":"81","DOI":"10.1016\/S0169-7161(06)26004-8","volume":"26","author":"NM Webb","year":"2006","unstructured":"Webb, N. M., Shavelson, R. J., & Haertel, E. H. (2006). Reliability coefficients and generalizability theory. Handbook of Statistics, 26, 81\u2013124.","journal-title":"Handbook of Statistics"},{"key":"9274_CR48","doi-asserted-by":"crossref","unstructured":"Webber, W., Moffat, A., & Zobel, J. (2008). Statistical power in retrieval experimentation. In ACM international conference on information and knowledge management, pp. 571\u2013580.","DOI":"10.1145\/1458082.1458158"},{"key":"9274_CR49","doi-asserted-by":"crossref","unstructured":"Yilmaz, E., Aslam, J. A., & Robertson, S. (2008). A new rank correlation coefficient for information retrieval. In International ACM SIGIR conference on research and development in information retrieval, pp. 587\u2013594.","DOI":"10.1145\/1390334.1390435"},{"key":"9274_CR50","doi-asserted-by":"crossref","unstructured":"Zobel, J. (1998). How reliable are the results of large-scale information retrieval experiments? In International ACM SIGIR conference on research and development in information retrieval, pp. 307\u2013314.","DOI":"10.1145\/290941.291014"}],"container-title":["Information Retrieval Journal"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-015-9274-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10791-015-9274-y\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-015-9274-y","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-015-9274-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,2]],"date-time":"2024-01-02T14:23:18Z","timestamp":1704205398000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10791-015-9274-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,12,23]]},"references-count":51,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2016,6]]}},"alternative-id":["9274"],"URL":"https:\/\/doi.org\/10.1007\/s10791-015-9274-y","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"value":"1386-4564","type":"print"},{"value":"1573-7659","type":"electronic"}],"subject":[],"published":{"date-parts":[[2015,12,23]]},"assertion":[{"value":"3 May 2015","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"14 October 2015","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 December 2015","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}