{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:00:10Z","timestamp":1784178010853,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":22,"publisher":"ACM","license":[{"start":{"date-parts":[[2012,10,29]],"date-time":"2012-10-29T00:00:00Z","timestamp":1351468800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2012,10,29]]},"DOI":"10.1145\/2396761.2396779","type":"proceedings-article","created":{"date-parts":[[2012,11,15]],"date-time":"2012-11-15T16:38:00Z","timestamp":1352997480000},"page":"105-114","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":23,"title":["An analysis of systematic judging errors in information retrieval"],"prefix":"10.1145","author":[{"given":"Gabriella","family":"Kazai","sequence":"first","affiliation":[{"name":"Microsoft Research, Cambridge, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nick","family":"Craswell","sequence":"additional","affiliation":[{"name":"Microsoft, Bellevue, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Emine","family":"Yilmaz","sequence":"additional","affiliation":[{"name":"Microsoft Research, Cambridge, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"S.M.M","family":"Tahaghoghi","sequence":"additional","affiliation":[{"name":"Microsoft, Bellevue, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2012,10,29]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.5555\/1996889.1996910"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390447"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.10286"},{"key":"e_1_3_2_1_4_1","first-page":"193","volume-title":"NIPS","author":"Burges C. J. C.","year":"2006","unstructured":"C. J. C. Burges , R. Ragno , and Q. V. Le . Learning to rank with nonsmooth cost functions . In NIPS , pages 193 -- 200 . MIT Press , 2006 . C. J. C. Burges, R. Ragno, and Q. V. Le. Learning to rank with nonsmooth cost functions. In NIPS, pages 193--200. MIT Press, 2006."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835540"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/1935826.1935847"},{"key":"e_1_3_2_1_7_1","first-page":"95","volume-title":"Proc. of the American Documentation Institute","author":"Cuadra C.","unstructured":"C. Cuadra and R. Katter . The relevance of relevance assessment . Proc. of the American Documentation Institute , page 95 -- 99 . C. Cuadra and R. Katter. The relevance of relevance assessment. Proc. of the American Documentation Institute, page 95--99."},{"key":"e_1_3_2_1_8_1","series-title":"Series C (Applied Statistics)","first-page":"20","volume-title":"Maximum likelihood estimation of observer error-rates using the em algorithm. Journal of the Royal Statistical Society","author":"Dawid A. P.","unstructured":"A. P. Dawid and A. M. Skene . Maximum likelihood estimation of observer error-rates using the em algorithm. Journal of the Royal Statistical Society . Series C (Applied Statistics) , pages 20 -- 28 . A. P. Dawid and A. M. Skene. Maximum likelihood estimation of observer error-rates using the em algorithm. Journal of the Royal Statistical Society. Series C (Applied Statistics), pages 20--28."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.5555\/1866696.1866723"},{"key":"e_1_3_2_1_10_1","volume-title":"The very large collection and web tracks","author":"Hawking D.","year":"2005","unstructured":"D. Hawking and N. Craswell . The very large collection and web tracks . In E. Voorhees and D. Harman, editors, TREC : Experiment and Evaluation in Information Retrieval. MIT Press , 2005 . D. Hawking and N. Craswell. The very large collection and web tracks. In E. Voorhees and D. Harman, editors, TREC: Experiment and Evaluation in Information Retrieval. MIT Press, 2005."},{"key":"e_1_3_2_1_11_1","volume-title":"Crowdsourcing: Why the Power of the Crowd Is Driving the Future of Business","author":"Howe J.","year":"2008","unstructured":"J. Howe . Crowdsourcing: Why the Power of the Crowd Is Driving the Future of Business . Crown Publishing Group , New York, NY, USA , 1 edition, 2008 . J. Howe. Crowdsourcing: Why the Power of the Crowd Is Driving the Future of Business. Crown Publishing Group, New York, NY, USA, 1 edition, 2008."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837906"},{"key":"e_1_3_2_1_13_1","volume-title":"TREC","author":"Jiyin He K. H.","year":"2009","unstructured":"K. H. Jiyin He , Krisztian Balog and E. Meij . Heuristic ranking and diversification of web documents . In TREC , 2009 . K. H. Jiyin He, Krisztian Balog and E. Meij. Heuristic ranking and diversification of web documents. In TREC, 2009."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2009947"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1458082.1458160"},{"key":"e_1_3_2_1_16_1","volume-title":"Perception and misperception of bias in human judgment. Trends in cognitive sciences, 11(1):37--43","author":"Pronin E.","year":"2007","unstructured":"E. Pronin . Perception and misperception of bias in human judgment. Trends in cognitive sciences, 11(1):37--43 , 2007 . E. Pronin. Perception and misperception of bias in human judgment. Trends in cognitive sciences, 11(1):37--43, 2007."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.v58:13"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010057"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291017"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/383952.383963"},{"key":"e_1_3_2_1_21_1","volume-title":"TREC: Experimentation and Evaluation in Information Retrieval","author":"Voorhees E. M.","year":"2005","unstructured":"E. M. Voorhees and D. K. Harman , editors . TREC: Experimentation and Evaluation in Information Retrieval . MIT Press , 2005 . E. M. Voorhees and D. K. Harman, editors. TREC: Experimentation and Evaluation in Information Retrieval. MIT Press, 2005."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/1871437.1871508"}],"event":{"name":"CIKM'12: 21st ACM International Conference on Information and Knowledge Management","location":"Maui Hawaii USA","acronym":"CIKM'12","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 21st ACM international conference on Information and knowledge management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2396761.2396779","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2396761.2396779","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T08:35:18Z","timestamp":1750235718000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2396761.2396779"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012,10,29]]},"references-count":22,"alternative-id":["10.1145\/2396761.2396779","10.1145\/2396761"],"URL":"https:\/\/doi.org\/10.1145\/2396761.2396779","relation":{},"subject":[],"published":{"date-parts":[[2012,10,29]]},"assertion":[{"value":"2012-10-29","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}