{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:00:49Z","timestamp":1784178049396,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":24,"publisher":"ACM","license":[{"start":{"date-parts":[[2008,7,20]],"date-time":"2008-07-20T00:00:00Z","timestamp":1216512000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2008,7,20]]},"DOI":"10.1145\/1390334.1390447","type":"proceedings-article","created":{"date-parts":[[2008,7,22]],"date-time":"2008-07-22T13:46:39Z","timestamp":1216734399000},"page":"667-674","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":129,"title":["Relevance assessment"],"prefix":"10.1145","author":[{"given":"Peter","family":"Bailey","sequence":"first","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nick","family":"Craswell","sequence":"additional","affiliation":[{"name":"Microsoft, Cambridge, United Kngdm"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ian","family":"Soboroff","sequence":"additional","affiliation":[{"name":"NIST, Gaithersburg, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Paul","family":"Thomas","sequence":"additional","affiliation":[{"name":"CSIRO, Canberra, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Arjen P.","family":"de Vries","sequence":"additional","affiliation":[{"name":"CWI, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Emine","family":"Yilmaz","sequence":"additional","affiliation":[{"name":"Microsoft Research, Cambridge, United Kngdm"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2008,7,20]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1162\/coli.07-034-R2"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148263"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1328964.1328969"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1008992.1009000"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1016\/0306-4573(92)90031-T"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.5555\/230386.230390"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1177\/001316446002000104"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291009"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1162\/089120104773633402"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1097-4571(199601)47:1%3C37::AID-ASI4%3E3.3.CO;2-I"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/345508.345545"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1108\/eb026616"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1016\/0020-0271(68)90029-6"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.5555\/2228065.2228070"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1515\/9783110871609"},{"key":"e_1_3_2_1_17_1","volume-title":"Nonparametric Statistics for the Behavioral Sciences","author":"Sigel S.","year":"1988","unstructured":"S. Sigel and N. J. Castellan . Nonparametric Statistics for the Behavioral Sciences . McGraw-Hill , 1988 . S. Sigel and N. J. Castellan. Nonparametric Statistics for the Behavioral Sciences. McGraw-Hill, 1988."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277908"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/564376.564433"},{"key":"e_1_3_2_1_20_1","volume-title":"Proc. ADCS","author":"Trotman A.","year":"2007","unstructured":"A. Trotman and D. Jenkinson . IR Evaluation Using Multiple Assessors per Topic . In Proc. ADCS , 2007 . A. Trotman and D. Jenkinson. IR Evaluation Using Multiple Assessors per Topic. In Proc. ADCS, 2007."},{"key":"e_1_3_2_1_21_1","volume-title":"Proc. SIGIR Workshop on Focused Retrieval","author":"Trotman A.","year":"2007","unstructured":"A. Trotman , N. Pharo , and D. Jenkinson . Can we at least agree on something ? In Proc. SIGIR Workshop on Focused Retrieval , 2007 . A. Trotman, N. Pharo, and D. Jenkinson. Can we at least agree on something? In Proc. SIGIR Workshop on Focused Retrieval, 2007."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291017"},{"key":"e_1_3_2_1_23_1","volume-title":"Overview of the Fifth Text REtrieval Conference (TREC-5). NIST","author":"Voorhees E. M.","year":"1996","unstructured":"E. M. Voorhees and D. Harman . Overview of the Fifth Text REtrieval Conference (TREC-5). NIST , 1996 . E. M. Voorhees and D. Harman. Overview of the Fifth Text REtrieval Conference (TREC-5). NIST, 1996."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/1183614.1183633"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390437"}],"event":{"name":"SIGIR '08: The 31st Annual International ACM SIGIR Conference","location":"Singapore Singapore","acronym":"SIGIR '08","sponsor":["ACM Association for Computing Machinery","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 31st annual international ACM SIGIR conference on Research and development in information retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1390334.1390447","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/1390334.1390447","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T14:47:20Z","timestamp":1750258040000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1390334.1390447"}},"subtitle":["are judges exchangeable and does it matter"],"short-title":[],"issued":{"date-parts":[[2008,7,20]]},"references-count":24,"alternative-id":["10.1145\/1390334.1390447","10.1145\/1390334"],"URL":"https:\/\/doi.org\/10.1145\/1390334.1390447","relation":{},"subject":[],"published":{"date-parts":[[2008,7,20]]},"assertion":[{"value":"2008-07-20","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}