{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T06:11:18Z","timestamp":1775283078263,"version":"3.50.1"},"reference-count":37,"publisher":"Elsevier BV","issue":"5","license":[{"start":{"date-parts":[[1992,9,1]],"date-time":"1992-09-01T00:00:00Z","timestamp":715305600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Information Processing &amp; Management"],"published-print":{"date-parts":[[1992,9]]},"DOI":"10.1016\/0306-4573(92)90031-t","type":"journal-article","created":{"date-parts":[[2002,10,8]],"date-time":"2002-10-08T18:05:40Z","timestamp":1034100340000},"page":"619-627","source":"Crossref","is-referenced-by-count":44,"title":["Variations in relevance judgments and the evaluation of retrieval performance"],"prefix":"10.1016","volume":"28","author":[{"given":"Robert","family":"Burgin","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"78","reference":[{"key":"10.1016\/0306-4573(92)90031-T_BIB1","series-title":"Proceedings of the American Documentation Institute","first-page":"383","article-title":"A comparison of relevance assessments by three types of evaluator","author":"Barhydt","year":"1964"},{"key":"10.1016\/0306-4573(92)90031-T_BIB2","series-title":"Technical Report No. 85-686","article-title":"Implementation of the SMART information retrieval system","author":"Buckley","year":"1985"},{"key":"10.1016\/0306-4573(92)90031-T_BIB3","doi-asserted-by":"crossref","first-page":"623","DOI":"10.1016\/0306-4573(91)90003-5","article-title":"The effect of indexing exhaustivity on retrieval performance","volume":"27","author":"Burgin","year":"1991","journal-title":"Information Processing & Management"},{"key":"10.1016\/0306-4573(92)90031-T_BIB4","first-page":"73","article-title":"Retrieval tests on 5 classification schemes-studies on patent classification systems II (1)","volume":"5","author":"Carpenter","year":"1978","journal-title":"International Classification"},{"key":"10.1016\/0306-4573(92)90031-T_BIB5","doi-asserted-by":"crossref","first-page":"30","DOI":"10.1002\/asi.5090190108","article-title":"Expected search length: a single measure of retrieval effectiveness based on the weak ordering action of retrieval systems","volume":"19","author":"Cooper","year":"1968","journal-title":"American Documentation"},{"key":"10.1016\/0306-4573(92)90031-T_BIB6","series-title":"Experimental studies of relevance judgments","author":"Cuadra","year":"1967"},{"key":"10.1016\/0306-4573(92)90031-T_BIB7","doi-asserted-by":"crossref","first-page":"291","DOI":"10.1108\/eb026436","article-title":"Opening the black box of \u2018relevance\u2019","volume":"23","author":"Cuadra","year":"19670","journal-title":"Journal of Documentation"},{"key":"10.1016\/0306-4573(92)90031-T_BIB8","doi-asserted-by":"crossref","first-page":"373","DOI":"10.1016\/0306-4573(88)90042-8","article-title":"Measuring relevance judgments","volume":"24","author":"Eisenberg","year":"1988","journal-title":"Information Processing & Management"},{"key":"10.1016\/0306-4573(92)90031-T_BIB9","first-page":"80","article-title":"Order effects: a preliminary study of the possible influence of presentation order on user judgments of document relevance","volume":"Volume 23","author":"Eisenberg","year":"1986"},{"key":"10.1016\/0306-4573(92)90031-T_BIB10","doi-asserted-by":"crossref","first-page":"293","DOI":"10.1002\/(SICI)1097-4571(198809)39:5<293::AID-ASI1>3.0.CO;2-I","article-title":"Order effects: a study of the possible influence of presentation order on user judgments of document relevance","volume":"39","author":"Eisenberg","year":"1988","journal-title":"Journal of the American Society for Information Science"},{"key":"10.1016\/0306-4573(92)90031-T_BIB11","first-page":"66","article-title":"Dichotomous relevance judgments and the evaluation of information systems","volume":"Volume 24","author":"Eisenberg","year":"1987"},{"key":"10.1016\/0306-4573(92)90031-T_BIB12","doi-asserted-by":"crossref","first-page":"320","DOI":"10.1002\/asi.5090070408","article-title":"Seven years of work on the organization of materials in the special library","volume":"7","author":"Gull","year":"1956","journal-title":"American Documentation"},{"key":"10.1016\/0306-4573(92)90031-T_BIB13","first-page":"21","article-title":"Effects of subjective expert evaluation of relevance on the performance parameters of a document-based information-retrieval system","volume":"13","author":"Kazhdan","year":"1979","journal-title":"Nauchno-Tekhnicheskaya Informatsiya, Seriya 2"},{"key":"10.1016\/0306-4573(92)90031-T_BIB14","doi-asserted-by":"crossref","first-page":"343","DOI":"10.1016\/0020-0271(68)90029-6","article-title":"Relevance assessments and retrieval system evaluation","volume":"4","author":"Lesk","year":"1968","journal-title":"Information Storage Retrieval"},{"key":"10.1016\/0306-4573(92)90031-T_BIB15","doi-asserted-by":"crossref","first-page":"179","DOI":"10.1177\/016555158901500306","article-title":"Minimizing information overload: the ranking of electronic messages","volume":"15","author":"Losee","year":"1989","journal-title":"Journal of Information Science"},{"key":"10.1016\/0306-4573(92)90031-T_BIB16","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/0306-4573(91)90027-J","article-title":"An analytic measure predicting information retrieval system performance","volume":"27","author":"Losee","year":"1991","journal-title":"Information Processing & Management"},{"key":"10.1016\/0306-4573(92)90031-T_BIB17","doi-asserted-by":"crossref","first-page":"265","DOI":"10.1177\/016555159001600407","article-title":"Relevance as an aid to evaluation in OPACs","volume":"16","author":"O'Brien","year":"1990","journal-title":"Journal of Information Science"},{"key":"10.1016\/0306-4573(92)90031-T_BIB18","doi-asserted-by":"crossref","first-page":"311","DOI":"10.1002\/asi.4630200405","article-title":"Some independent agreements and resolved disagreements about answer-providing documents","volume":"20","author":"O'Connor","year":"1969","journal-title":"American Documentation"},{"key":"10.1016\/0306-4573(92)90031-T_BIB19","doi-asserted-by":"crossref","first-page":"126","DOI":"10.1002\/asi.5090120208","article-title":"Comparisons of four types of lexical indicators of content","volume":"12","author":"Rath","year":"1961","journal-title":"American Documentation"},{"key":"10.1016\/0306-4573(92)90031-T_BIB20","first-page":"63","article-title":"Evaluation of information systems and services","volume":"2","author":"Rees","year":"1967","journal-title":"Annual Review of Information Science and Technology"},{"key":"10.1016\/0306-4573(92)90031-T_BIB21","series-title":"A field experimental approach to the study of relevance assessments in relation to document searching, final report","author":"Rees","year":"1967"},{"key":"10.1016\/0306-4573(92)90031-T_BIB22","doi-asserted-by":"crossref","first-page":"235","DOI":"10.1002\/(SICI)1097-4571(198807)39:4<235::AID-ASI3>3.0.CO;2-H","article-title":"Performance measures for information retrieval systems \u2014 an experimental approach","volume":"39","author":"Regazzi","year":"1988","journal-title":"Journal of the American Society for Information Science"},{"issue":"3484","key":"10.1016\/0306-4573(92)90031-T_BIB23","doi-asserted-by":"crossref","first-page":"1004","DOI":"10.1126\/science.134.3484.1004","article-title":"Relative effectiveness of document titles and abstracts for determining the relevance of documents","volume":"134","author":"Resnick","year":"1961","journal-title":"Science"},{"key":"10.1016\/0306-4573(92)90031-T_BIB24","first-page":"157","article-title":"Psychometric measurement and information retrieval","volume":"23","author":"Rorvig","year":"1988","journal-title":"Annual Review of Information Science and Technology"},{"key":"10.1016\/0306-4573(92)90031-T_BIB25","series-title":"The SMART retrieval system \u2014 experiments in automatic document processing","year":"1971"},{"key":"10.1016\/0306-4573(92)90031-T_BIB26","doi-asserted-by":"crossref","first-page":"513","DOI":"10.1016\/0306-4573(88)90021-0","article-title":"Term-weighting approaches in automatic text retrieval","volume":"24","author":"Salton","year":"1988","journal-title":"Information Processing & Management"},{"key":"10.1016\/0306-4573(92)90031-T_BIB27","first-page":"293","article-title":"Comparative effects of titles, abstracts and full texts on relevance judgments","volume":"Volume 6","author":"Saracevic","year":"1969"},{"key":"10.1016\/0306-4573(92)90031-T_BIB28","series-title":"Introduction to information science","first-page":"111","article-title":"The concept of relevance in information science: a historical review","author":"Saracevic","year":"1970"},{"key":"10.1016\/0306-4573(92)90031-T_BIB29","first-page":"79","article-title":"Relevance: a review of the literature and a framework for thinking on the notion in information science","volume":"6","author":"Saracevic","year":"1976","journal-title":"Advances in Librarianship"},{"key":"10.1016\/0306-4573(92)90031-T_BIB30","doi-asserted-by":"crossref","first-page":"346","DOI":"10.1002\/asi.4630370511","article-title":"On the foundation of evaluation","volume":"37","author":"Shaw","year":"1986","journal-title":"Journal of the American Society for Information Science"},{"key":"10.1016\/0306-4573(92)90031-T_BIB31","doi-asserted-by":"crossref","first-page":"19","DOI":"10.1016\/0306-4573(86)90006-3","article-title":"An investigation of document partitions","volume":"22","author":"Shaw","year":"1986","journal-title":"Information Processing and Management"},{"key":"10.1016\/0306-4573(92)90031-T_BIB32","doi-asserted-by":"crossref","first-page":"339","DOI":"10.1016\/0306-4573(90)90095-J","article-title":"An investigation of document structures","volume":"26","author":"Shaw","year":"1990","journal-title":"Information Processing & Management"},{"key":"10.1016\/0306-4573(92)90031-T_BIB33","doi-asserted-by":"crossref","first-page":"705","DOI":"10.1016\/0306-4573(90)90047-6","article-title":"Subject indexing and citation indexing \u2014 part II: an evaluation and comparison","volume":"26","author":"Shaw","year":"1990","journal-title":"Information Processing & Management"},{"key":"10.1016\/0306-4573(92)90031-T_BIB34","series-title":"The cystic fibrosis data base: content and research opportunities","author":"Shaw","year":"1990"},{"key":"10.1016\/0306-4573(92)90031-T_BIB35","doi-asserted-by":"crossref","first-page":"393","DOI":"10.1108\/eb026588","article-title":"Automatic indexing","volume":"30","author":"Sparck Jones","year":"1974","journal-title":"Journal of Documentation"},{"key":"10.1016\/0306-4573(92)90031-T_BIB36","series-title":"Information retrieval","author":"Van Rijsbergen","year":"1979"},{"key":"10.1016\/0306-4573(92)90031-T_BIB37","series-title":"Technical Report No. 8902","article-title":"The cystic fibrosis data base","author":"Wood","year":"1990"}],"container-title":["Information Processing &amp; Management"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:030645739290031T?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:030645739290031T?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2019,4,10]],"date-time":"2019-04-10T09:24:22Z","timestamp":1554888262000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/030645739290031T"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[1992,9]]},"references-count":37,"journal-issue":{"issue":"5","published-print":{"date-parts":[[1992,9]]}},"alternative-id":["030645739290031T"],"URL":"https:\/\/doi.org\/10.1016\/0306-4573(92)90031-t","relation":{},"ISSN":["0306-4573"],"issn-type":[{"value":"0306-4573","type":"print"}],"subject":[],"published":{"date-parts":[[1992,9]]}}}