{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T06:19:51Z","timestamp":1775283591396,"version":"3.50.1"},"reference-count":8,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2016,6,1]],"date-time":"2016-06-01T00:00:00Z","timestamp":1464739200000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval J"],"published-print":{"date-parts":[[2016,6]]},"DOI":"10.1007\/s10791-016-9281-7","type":"journal-article","created":{"date-parts":[[2016,6,10]],"date-time":"2016-06-10T09:15:07Z","timestamp":1465550107000},"page":"225-229","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":13,"title":["Information retrieval evaluation using test collections"],"prefix":"10.1007","volume":"19","author":[{"given":"Falk","family":"Scholer","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Diane","family":"Kelly","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ben","family":"Carterette","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2016,6,10]]},"reference":[{"key":"9281_CR1","volume-title":"Readings in information retrieval","author":"C Cleverdon","year":"1997","unstructured":"Cleverdon, C. (1997). The Cranfield tests on index language devices. In K. Sp\u00e4rckJones & P. Willett (Eds.), Readings in information retrieval. San Francisco: Morgan Kaufmann."},{"key":"9281_CR2","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-015-9275-x","author":"D Demeester","year":"2016","unstructured":"Demeester, D., Aly, R., Hiemstra, D., Nguyen, D., & Develder, C. (2016). Predicting relevance based on assessor disagreement: Analysis and practical applications for search evaluation. Information Retrieval Journal. doi: 10.1007\/s10791-015-9275-x .","journal-title":"Information Retrieval Journal"},{"issue":"4","key":"9281_CR3","doi-asserted-by":"crossref","first-page":"439","DOI":"10.1177\/0165551507086989","volume":"34","author":"S Robertson","year":"2008","unstructured":"Robertson, S. (2008). On the history of evaluation in IR. Journal of Information Science, 34(4), 439\u2013456.","journal-title":"Journal of Information Science"},{"key":"9281_CR4","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-015-9273-z","author":"T Sakai","year":"2016","unstructured":"Sakai, T. (2016). Topic set size design. Information Retrieval Journal. doi: 10.1007\/s10791-015-9273-z .","journal-title":"Information Retrieval Journal"},{"key":"9281_CR5","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-015-9276-9","author":"T Samar","year":"2016","unstructured":"Samar, T., Bellogin, A., & de Vries, A. (2016). The strange case of reproducibility vs. representativeness in contextual suggestion test collections. Information Retrieval Journal. doi: 10.1007\/s10791-015-9276-9 .","journal-title":"Information Retrieval Journal"},{"key":"9281_CR6","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-015-9274-y","author":"J Urbano","year":"2016","unstructured":"Urbano, J. (2016). Measures of test collection reliability and their robustness to statistical assumptions. Information Retrieval Journal. doi: 10.1007\/s10791-015-9274-y .","journal-title":"Information Retrieval Journal"},{"issue":"11","key":"9281_CR7","doi-asserted-by":"crossref","first-page":"51","DOI":"10.1145\/1297797.1297822","volume":"50","author":"EM Voorhees","year":"2007","unstructured":"Voorhees, E. M. (2007). TREC: Continuing information retrieval\u2019s tradition of experimentation. Communications of the ACM, 50(11), 51\u201354.","journal-title":"Communications of the ACM"},{"key":"9281_CR8","volume-title":"TREC: Experiment and evaluation in information retrieval","author":"EM Voorhees","year":"2005","unstructured":"Voorhees, E. M., & Harman, D. (2005). TREC: Experiment and evaluation in information retrieval. Cambridge, MA: MIT Press."}],"container-title":["Information Retrieval Journal"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-016-9281-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10791-016-9281-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-016-9281-7","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2017,6,24]],"date-time":"2017-06-24T16:05:29Z","timestamp":1498320329000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10791-016-9281-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,6]]},"references-count":8,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2016,6]]}},"alternative-id":["9281"],"URL":"https:\/\/doi.org\/10.1007\/s10791-016-9281-7","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"value":"1386-4564","type":"print"},{"value":"1573-7659","type":"electronic"}],"subject":[],"published":{"date-parts":[[2016,6]]}}}