{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T06:11:42Z","timestamp":1775283102669,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":17,"publisher":"ACM","license":[{"start":{"date-parts":[[2009,7,19]],"date-time":"2009-07-19T00:00:00Z","timestamp":1247961600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2009,7,19]]},"DOI":"10.1145\/1571941.1572019","type":"proceedings-article","created":{"date-parts":[[2009,7,28]],"date-time":"2009-07-28T12:45:28Z","timestamp":1248785128000},"page":"452-459","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":29,"title":["Towards methods for the collective gathering and quality control of relevance assessments"],"prefix":"10.1145","author":[{"given":"Gabriella","family":"Kazai","sequence":"first","affiliation":[{"name":"Microsoft Research, Cambridge, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Natasa","family":"Milic-Frayling","sequence":"additional","affiliation":[{"name":"Microsoft Research, Cambridge, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jamie","family":"Costello","sequence":"additional","affiliation":[{"name":"Microsoft Research, Cambridge, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2009,7,19]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390447"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.2307\/2390752"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277934"},{"key":"e_1_3_2_1_4_1","volume-title":"Overview of the INEX 2007 ad hoc track. In Proc. of INEX'07. 1--22","author":"Fuhr N.","unstructured":"Fuhr , N. , Kamps , J. , Lalmas , M. , Malik , S. , and Trotman , A . 2007 . Overview of the INEX 2007 ad hoc track. In Proc. of INEX'07. 1--22 . Fuhr, N., Kamps, J., Lalmas, M., Malik, S., and Trotman, A. 2007. Overview of the INEX 2007 ad hoc track. In Proc. of INEX'07. 1--22."},{"key":"e_1_3_2_1_5_1","volume-title":"Proc. of INEX'08. LNCS","author":"Kazai G.","unstructured":"Kazai , G. , Doucet , A. , Landoni , M. 2009. Overview of the INEX 2008 Book Track . In Proc. of INEX'08. LNCS Vol. 5613 , Springer . Kazai, G., Doucet, A., Landoni, M. 2009. Overview of the INEX 2008 Book Track. In Proc. of INEX'08. LNCS Vol. 5613, Springer."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416951"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/1008992.1009001"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/383952.383961"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1002\/1097-4571(2000)9999:9999%3C::AID-ASI1564%3E3.3.CO;2-C"},{"key":"e_1_3_2_1_10_1","volume-title":"Pre-Proc. of INEX'06","author":"Trotman A.","year":"2006","unstructured":"Trotman , A. , Pharo , N. & Lehtonen ( 2006 ). XML-IR users and use cases . In Pre-Proc. of INEX'06 , 274--286. Trotman, A., Pharo, N.&Lehtonen (2006). XML-IR users and use cases. In Pre-Proc. of INEX'06, 274--286."},{"key":"e_1_3_2_1_11_1","volume-title":"Proc. of ADCS.","author":"Trotman A.","unstructured":"Trotman , A. and Jenkinson , D . 2007. IR evaluation using multiple assessors per topic . In Proc. of ADCS. Trotman, A. and Jenkinson, D. 2007. IR evaluation using multiple assessors per topic. In Proc. of ADCS."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/1378704.1378719"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/985692.985733"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/345508.345577"},{"key":"e_1_3_2_1_15_1","volume-title":"2005 TREC: Experiment and Evaluation in Information Retrieval","author":"Voorhees E. M.","unstructured":"Voorhees , E. M. and Harman , D. K . 2005 TREC: Experiment and Evaluation in Information Retrieval . The MIT Press . Voorhees, E. M. and Harman, D. K. 2005 TREC: Experiment and Evaluation in Information Retrieval. The MIT Press."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390437"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR '09: The 32nd International ACM SIGIR conference on research and development in Information Retrieval","location":"Boston MA USA","acronym":"SIGIR '09","sponsor":["ACM Association for Computing Machinery","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 32nd international ACM SIGIR conference on Research and development in information retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1571941.1572019","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/1571941.1572019","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T12:23:33Z","timestamp":1750249413000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1571941.1572019"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2009,7,19]]},"references-count":17,"alternative-id":["10.1145\/1571941.1572019","10.1145\/1571941"],"URL":"https:\/\/doi.org\/10.1145\/1571941.1572019","relation":{},"subject":[],"published":{"date-parts":[[2009,7,19]]},"assertion":[{"value":"2009-07-19","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}