{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T01:00:52Z","timestamp":1774400452253,"version":"3.50.1"},"reference-count":17,"publisher":"Elsevier BV","issue":"5","license":[{"start":{"date-parts":[[2000,9,1]],"date-time":"2000-09-01T00:00:00Z","timestamp":967766400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Information Processing &amp; Management"],"published-print":{"date-parts":[[2000,9]]},"DOI":"10.1016\/s0306-4573(00)00010-8","type":"journal-article","created":{"date-parts":[[2002,7,26]],"date-time":"2002-07-26T02:36:39Z","timestamp":1027650999000},"page":"697-716","source":"Crossref","is-referenced-by-count":261,"title":["Variations in relevance judgments and the measurement of retrieval effectiveness"],"prefix":"10.1016","volume":"36","author":[{"given":"Ellen M.","family":"Voorhees","sequence":"first","affiliation":[]}],"member":"78","reference":[{"issue":"12","key":"10.1016\/S0306-4573(00)00010-8_BIB1","doi-asserted-by":"crossref","first-page":"29","DOI":"10.1145\/138859.138861","article-title":"Information filtering and information retrieval: two sides of the same coin?","volume":"35","author":"Belkin","year":"1992","journal-title":"Communications of the ACM"},{"issue":"5","key":"10.1016\/S0306-4573(00)00010-8_BIB2","doi-asserted-by":"crossref","first-page":"619","DOI":"10.1016\/0306-4573(92)90031-T","article-title":"Variations in relevance judgments and the evaluation of retrieval performance","volume":"28","author":"Burgin","year":"1992","journal-title":"Information Processing and Management"},{"key":"10.1016\/S0306-4573(00)00010-8_BIB3","unstructured":"Cleverdon, C. W. (1970). The effect of variations in relevance assessments in comparative experimental tests of index languages (Cranfield Library Report No. 3). Cranfield, UK: Cranfield Institute of Technology"},{"key":"10.1016\/S0306-4573(00)00010-8_BIB4","unstructured":"Cleverdon, C. W., Mills, J., & Keen, E. M. (1968). Factors determining the performance of indexing systems. Two volumes. Cranfield, England"},{"key":"10.1016\/S0306-4573(00)00010-8_BIB5","series-title":"Proceedings of the Sixth Text REtrieval Conference (TREC-6)","first-page":"303","article-title":"Passage-based refinement (MultiText experiments for TREC-6)","author":"Cormack","year":"1998"},{"issue":"4","key":"10.1016\/S0306-4573(00)00010-8_BIB6","doi-asserted-by":"crossref","first-page":"291","DOI":"10.1108\/eb026436","article-title":"Opening the black box of relevance","volume":"23","author":"Cuadra","year":"1967","journal-title":"Journal of Documentation"},{"issue":"4","key":"10.1016\/S0306-4573(00)00010-8_BIB7","doi-asserted-by":"crossref","first-page":"411","DOI":"10.1016\/0306-4573(93)90037-E","article-title":"The first Text REtrieval Conference (TREC-1), Rockville, MD, USA, 4\u20136 November, 1992","volume":"29","author":"Harman","year":"1993","journal-title":"Information Processing and Management"},{"issue":"1","key":"10.1016\/S0306-4573(00)00010-8_BIB8","doi-asserted-by":"crossref","first-page":"37","DOI":"10.1002\/(SICI)1097-4571(199601)47:1<37::AID-ASI4>3.0.CO;2-3","article-title":"Variations in relevance assessments and the measurement of retrieval effectiveness","volume":"47","author":"Harter","year":"1996","journal-title":"Journal of the American Society for Information Science"},{"key":"10.1016\/S0306-4573(00)00010-8_BIB9","series-title":"Proceedings of the Sixth Text REtrieval Conference (TREC-6)","first-page":"207","article-title":"TREC-6 English and Chinese retrieval experiments using PIRCS","author":"Kwok","year":"1998"},{"key":"10.1016\/S0306-4573(00)00010-8_BIB10","doi-asserted-by":"crossref","first-page":"343","DOI":"10.1016\/0020-0271(68)90029-6","article-title":"Relevance assessments and retrieval system evaluation","volume":"4","author":"Lesk","year":"1969","journal-title":"Information Storage and Retrieval"},{"key":"10.1016\/S0306-4573(00)00010-8_BIB11","series-title":"The SMART retrieval system: experiments in automatic document processing","year":"1971"},{"key":"10.1016\/S0306-4573(00)00010-8_BIB12","first-page":"3","article-title":"Relevance and information behavior","volume":"29","author":"Schamber","year":"1994","journal-title":"Annual Review of Information Science and Technology"},{"key":"10.1016\/S0306-4573(00)00010-8_BIB13","doi-asserted-by":"crossref","first-page":"393","DOI":"10.1108\/eb026588","article-title":"Automatic indexing","volume":"30","author":"Sparck Jones","year":"1974","journal-title":"Journal of Documentation"},{"key":"10.1016\/S0306-4573(00)00010-8_BIB14","series-title":"Information retrieval experiment","author":"Sparck Jones","year":"1981"},{"key":"10.1016\/S0306-4573(00)00010-8_BIB15","first-page":"367","article-title":"Kendall\u2019s tau","volume":"Vol.4","author":"Stuart","year":"1983"},{"issue":"2","key":"10.1016\/S0306-4573(00)00010-8_BIB16","doi-asserted-by":"crossref","first-page":"69","DOI":"10.1002\/asi.5090160204","article-title":"A note on the pseudomathematics of relevance","volume":"16","author":"Taube","year":"1965","journal-title":"American Documentation"},{"issue":"3","key":"10.1016\/S0306-4573(00)00010-8_BIB17","doi-asserted-by":"crossref","first-page":"273","DOI":"10.1016\/0306-4573(95)00061-5","article-title":"Relevance judgments for assessing recall","volume":"32","author":"Wallis","year":"1996","journal-title":"Information Processing & Management"}],"container-title":["Information Processing &amp; Management"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0306457300000108?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0306457300000108?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2019,4,14]],"date-time":"2019-04-14T12:23:03Z","timestamp":1555244583000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0306457300000108"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2000,9]]},"references-count":17,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2000,9]]}},"alternative-id":["S0306457300000108"],"URL":"https:\/\/doi.org\/10.1016\/s0306-4573(00)00010-8","relation":{},"ISSN":["0306-4573"],"issn-type":[{"value":"0306-4573","type":"print"}],"subject":[],"published":{"date-parts":[[2000,9]]}}}