{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:00:10Z","timestamp":1784178010091,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":24,"publisher":"ACM","license":[{"start":{"date-parts":[[2011,7,24]],"date-time":"2011-07-24T00:00:00Z","timestamp":1311465600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2011,7,24]]},"DOI":"10.1145\/2009916.2010057","type":"proceedings-article","created":{"date-parts":[[2011,7,26]],"date-time":"2011-07-26T14:09:37Z","timestamp":1311689377000},"page":"1063-1072","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":52,"title":["Quantifying test collection quality based on the consistency of relevance judgements"],"prefix":"10.1145","author":[{"given":"Falk","family":"Scholer","sequence":"first","affiliation":[{"name":"RMIT University, Melbourne, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Andrew","family":"Turpin","sequence":"additional","affiliation":[{"name":"University of Melbourne, Melbourne, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mark","family":"Sanderson","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2011,7,24]]},"reference":[{"key":"e_1_3_2_1_1_1","first-page":"15","volume-title":"Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation","author":"Alonso O.","year":"2009","unstructured":"O. Alonso and S. Mizzaro . Can we get rid of TREC assessors? Using mechanical turk for relevance assessment . In Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation , pages 15 -- 16 , 2009 . O. Alonso and S. Mizzaro. Can we get rid of TREC assessors? Using mechanical turk for relevance assessment. In Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation, pages 15--16, 2009."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390447"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1099554.1099733"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835540"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835540"},{"key":"e_1_3_2_1_6_1","first-page":"687","volume-title":"Proceedings of the International Conference on Scientific Information -- Two Volumes","author":"Cleverdon C.","year":"1959","unstructured":"C. Cleverdon . The evaluation of systems used in information retrieval (1958: Washington) . In Proceedings of the International Conference on Scientific Information -- Two Volumes , pages 687 -- 698 . Washington : National Academy of Sciences, National Research Council , 1959 . C. Cleverdon. The evaluation of systems used in information retrieval (1958: Washington). In Proceedings of the International Conference on Scientific Information -- Two Volumes, pages 687--698. Washington : National Academy of Sciences, National Research Council, 1959."},{"key":"e_1_3_2_1_7_1","volume-title":"The effect of variations in relevance assessments in comparative experimental tests of index languages","author":"Cleverdon C. W.","year":"1970","unstructured":"C. W. Cleverdon . The effect of variations in relevance assessments in comparative experimental tests of index languages . Cranfield Library Report 3, Cranfield Institute of Technology , 1970 . C. W. Cleverdon. The effect of variations in relevance assessments in comparative experimental tests of index languages. Cranfield Library Report 3, Cranfield Institute of Technology, 1970."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291009"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.5090070408"},{"key":"e_1_3_2_1_10_1","volume-title":"The TREC test collection","author":"Harman D. K.","year":"2005","unstructured":"D. K. Harman . The TREC test collection . In E. M. Voorhees and D. K. Harman, editors, TREC: experiment and evaluation in information retrieval. MIT Press , 2005 . D. K. Harman. The TREC test collection. In E. M. Voorhees and D. K. Harman, editors, TREC: experiment and evaluation in information retrieval. MIT Press, 2005."},{"key":"e_1_3_2_1_11_1","first-page":"1","volume-title":"Overview of the TREC 2009 legal track. In Proceedings of the 18th Text REtrieval Conference","author":"Hedin B.","year":"2010","unstructured":"B. Hedin , D. Oard , S. Tomlinson , and J. Baron . Overview of the TREC 2009 legal track. In Proceedings of the 18th Text REtrieval Conference , pages 4: 1 -- 40 , 2010 . B. Hedin, D. Oard, S. Tomlinson, and J. Baron. Overview of the TREC 2009 legal track. In Proceedings of the 18th Text REtrieval Conference, pages 4:1--40, 2010."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1016\/0020-0271(68)90002-8"},{"key":"e_1_3_2_1_13_1","volume-title":"Relevance assessments and retrieval system evaluation. Information storage and retrieval, 4(4):343--359","author":"Lesk M.","year":"1968","unstructured":"M. Lesk and G. Salton . Relevance assessments and retrieval system evaluation. Information storage and retrieval, 4(4):343--359 , 1968 . M. Lesk and G. Salton. Relevance assessments and retrieval system evaluation. Information storage and retrieval, 4(4):343--359, 1968."},{"key":"e_1_3_2_1_14_1","first-page":"60","volume-title":"Proceedings of the Australasian Document Computing Symposium","author":"Sanderson M.","year":"2010","unstructured":"M. Sanderson , F. Scholer , and A. Turpin . Relatively relevant: Assessor shift in document judgements . In Proceedings of the Australasian Document Computing Symposium , pages 60 -- 67 , Melbourne, Australia , 2010 . M. Sanderson, F. Scholer, and A. Turpin. Relatively relevant: Assessor shift in document judgements. In Proceedings of the Australasian Document Computing Symposium, pages 60--67, Melbourne, Australia, 2010."},{"key":"e_1_3_2_1_15_1","volume-title":"Handbook of parametric and nonparametric statistical procedures","author":"Sheskin D.","year":"2007","unstructured":"D. Sheskin . Handbook of parametric and nonparametric statistical procedures , 4 th ed. CRC Press , 2007 . D. Sheskin. Handbook of parametric and nonparametric statistical procedures, 4th ed. CRC Press, 2007.","edition":"4"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/1321440.1321528"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1108\/eb026215"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(00)00010-8"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(00)00010-8"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/564376.564432"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/1458082.1458158"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/1871437.1871508"},{"key":"e_1_3_2_1_23_1","volume-title":"Managing Gigabytes: Compressing and Indexing Documents and Images","author":"Witten I.","year":"1999","unstructured":"I. Witten , A. Moffat , and T. Bell . Managing Gigabytes: Compressing and Indexing Documents and Images . Morgan Kaufmann Publishers , 2 nd edition, 1999 . I. Witten, A. Moffat, and T. Bell. Managing Gigabytes: Compressing and Indexing Documents and Images. Morgan Kaufmann Publishers, 2nd edition, 1999.","edition":"2"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR '11: The 34th International ACM SIGIR conference on research and development in Information Retrieval","location":"Beijing China","acronym":"SIGIR '11","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 34th international ACM SIGIR conference on Research and development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2009916.2010057","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2009916.2010057","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T11:06:28Z","timestamp":1750244788000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2009916.2010057"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2011,7,24]]},"references-count":24,"alternative-id":["10.1145\/2009916.2010057","10.1145\/2009916"],"URL":"https:\/\/doi.org\/10.1145\/2009916.2010057","relation":{},"subject":[],"published":{"date-parts":[[2011,7,24]]},"assertion":[{"value":"2011-07-24","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}