{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,12]],"date-time":"2026-04-12T15:18:25Z","timestamp":1776007105124,"version":"3.50.1"},"reference-count":19,"publisher":"Elsevier BV","issue":"4","license":[{"start":{"date-parts":[[1968,12,1]],"date-time":"1968-12-01T00:00:00Z","timestamp":-34214400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Information Storage and Retrieval"],"published-print":{"date-parts":[[1968,12]]},"DOI":"10.1016\/0020-0271(68)90029-6","type":"journal-article","created":{"date-parts":[[2002,10,8]],"date-time":"2002-10-08T18:05:40Z","timestamp":1034100340000},"page":"343-359","source":"Crossref","is-referenced-by-count":84,"title":["Relevance assessments and retrieval system evaluation"],"prefix":"10.1016","volume":"4","author":[{"given":"M.E.","family":"Lesk","sequence":"first","affiliation":[]},{"given":"G.","family":"Salton","sequence":"additional","affiliation":[]}],"member":"78","reference":[{"key":"10.1016\/0020-0271(68)90029-6_BIB1","article-title":"Test Results","volume":"Vol. 2","author":"Cleverdon","year":"1966"},{"key":"10.1016\/0020-0271(68)90029-6_BIB2","series-title":"Report ESD-TR-66-405","article-title":"Study and test of a methodology for laboratory evaluation of message retrieval systems","author":"Giuuano","year":"1966"},{"key":"10.1016\/0020-0271(68)90029-6_BIB3_1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/321439.321441","article-title":"Computer evaluation of indexing and text processing","volume":"15","author":"Salton","year":"1968","journal-title":"Journal of the ACM"},{"key":"10.1016\/0020-0271(68)90029-6_BIB3_2","doi-asserted-by":"crossref","first-page":"8","DOI":"10.1145\/321439.321441","article-title":"Computer evaluation of indexing and text processing","volume":"15","author":"Salton","year":"1968","journal-title":"Journal of the ACM"},{"key":"10.1016\/0020-0271(68)90029-6_BIB4_1","article-title":"Experimental studies of relevance judgments: Final Report, Report TM-3520","volume":"Vol. 1","author":"Cuadra","year":"1967","journal-title":"Project Summary"},{"key":"10.1016\/0020-0271(68)90029-6_BIB4_2","article-title":"Experimental studies of relevance judgments: Final Report, Report TM-3520","volume":"Vol. 2","author":"Cuadra","year":"1967","journal-title":"Description of Individual Studies, System Development Corp."},{"key":"10.1016\/0020-0271(68)90029-6_BIB5","first-page":"109","article-title":"Implications of Test Procedures, in Information Retrieval in Action","author":"Fairthorne","year":"1963"},{"key":"10.1016\/0020-0271(68)90029-6_BIB6","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1002\/asi.5090180310","article-title":"Relevance disagreements and unclear request forms","volume":"18","author":"O'Connor","year":"1967","journal-title":"Am. Docum."},{"key":"10.1016\/0020-0271(68)90029-6_BIB7","doi-asserted-by":"crossref","first-page":"2","DOI":"10.1002\/asi.5090190216","article-title":"Some questions concerning information need","volume":"19","author":"O'Connor","year":"1968","journal-title":"Am. Docum."},{"key":"10.1016\/0020-0271(68)90029-6_BIB8","series-title":"Proceedings of the 26th Annual Meeting, American Documentation Institute","article-title":"Is relevance an adequate criterion in retrieval system evaluation","author":"Doyle","year":"1963"},{"issue":"2","key":"10.1016\/0020-0271(68)90029-6_BIB9","doi-asserted-by":"crossref","first-page":"69","DOI":"10.1002\/asi.5090160204","article-title":"A note on the pseudomathematics of relevance","volume":"16","author":"Taube","year":"1965","journal-title":"Am. Docum."},{"key":"10.1016\/0020-0271(68)90029-6_BIB10","unstructured":"E. D. Dym: Relevance Predictability\u2014Investigation, Background and Procedures, in Electronic Handling of Information: Testing and Evaluation, Eds.: A. Kent et al., pp. 175\u2013185. Thompson Book Co., Washington."},{"key":"10.1016\/0020-0271(68)90029-6_BIB11","unstructured":"D. L. Shirey and M. Kurfeerst: Relevance Predictability\u2014Data Reduction, in Electronic Handling of Information: Testing and Evaluation, Eds.: A. Kent et al., pp. 175\u2013185. Thompson Book Co., Washington."},{"issue":"No. 4","key":"10.1016\/0020-0271(68)90029-6_BIB12","doi-asserted-by":"crossref","DOI":"10.1108\/eb026436","article-title":"Opening the black box of relevance","volume":"23","author":"Cuadra","year":"1967","journal-title":"J. Docum."},{"key":"10.1016\/0020-0271(68)90029-6_BIB13","article-title":"Evaluation of information systems and services","volume":"Vol. 2","author":"Rees","year":"1967"},{"key":"10.1016\/0020-0271(68)90029-6_BIB14","series-title":"Final Report to the National Science Foundation","article-title":"A field experimental approach to the study of relevance assessments in relation to document searching","author":"Rees","year":"1967"},{"key":"10.1016\/0020-0271(68)90029-6_BIB15","series-title":"Scientific Reports on the SMART System to the National Science Foundation, Nos. ISR-11, ISR-12, ISR-13","author":"Salton","year":"1966-1967-1968"},{"key":"10.1016\/0020-0271(68)90029-6_BIB16","series-title":"Proceedings IFIP Congress 1968","article-title":"Search and retrieval experiments in real-time information retrieval","author":"Salton","year":"1968"},{"key":"10.1016\/0020-0271(68)90029-6_BIB17","series-title":"Technical Report No. 68-11","article-title":"A comparison between manual and automatic indexing methods","author":"Salton","year":"1968"}],"container-title":["Information Storage and Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:0020027168900296?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:0020027168900296?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2019,4,10]],"date-time":"2019-04-10T08:54:36Z","timestamp":1554886476000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/0020027168900296"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[1968,12]]},"references-count":19,"journal-issue":{"issue":"4","published-print":{"date-parts":[[1968,12]]}},"alternative-id":["0020027168900296"],"URL":"https:\/\/doi.org\/10.1016\/0020-0271(68)90029-6","relation":{},"ISSN":["0020-0271"],"issn-type":[{"value":"0020-0271","type":"print"}],"subject":[],"published":{"date-parts":[[1968,12]]}}}