{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T04:19:44Z","timestamp":1750306784340,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":26,"publisher":"ACM","license":[{"start":{"date-parts":[[2013,10,27]],"date-time":"2013-10-27T00:00:00Z","timestamp":1382832000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2013,10,27]]},"DOI":"10.1145\/2505515.2505716","type":"proceedings-article","created":{"date-parts":[[2013,11,6]],"date-time":"2013-11-06T19:30:29Z","timestamp":1383766229000},"page":"699-708","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":12,"title":["User intent and assessor disagreement in web search evaluation"],"prefix":"10.1145","author":[{"given":"Gabriella","family":"Kazai","sequence":"first","affiliation":[{"name":"Microsoft Research, Cambridge, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Emine","family":"Yilmaz","sequence":"additional","affiliation":[{"name":"Microsoft Research, Cambridge, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nick","family":"Craswell","sequence":"additional","affiliation":[{"name":"Microsoft, Bellevue, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"S.M.M.","family":"Tahaghoghi","sequence":"additional","affiliation":[{"name":"Microsoft, Bellevue, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2013,10,27]]},"reference":[{"doi-asserted-by":"publisher","key":"e_1_3_2_1_1_1","DOI":"10.1145\/1498759.1498824"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_2_1","DOI":"10.1007\/978-3-642-20161-5_16"},{"key":"e_1_3_2_1_3_1","first-page":"15","volume-title":"Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation","author":"Alonso O.","year":"2009","unstructured":"O. Alonso and S. Mizzaro. Can we get rid of TREC assessors? using Mechanical Turk for relevance assessment. In S. Geva, J. Kamps, C. Peters, T. Sakai, A. Trotman, and E. Voorhees, editors, Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation, pages 15--16, 2009."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_4_1","DOI":"10.5555\/1793274.1793281"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_5_1","DOI":"10.1108\/eb050097"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_6_1","DOI":"10.1145\/1341531.1341545"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_7_1","DOI":"10.1145\/1924421.1924442"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_8_1","DOI":"10.1145\/1753326.1753688"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_9_1","DOI":"10.1145\/1718487.1718510"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_10_1","DOI":"10.5555\/1866696.1866723"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_11_1","DOI":"10.1145\/2063576.2063618"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_12_1","DOI":"10.5555\/1481457"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_13_1","DOI":"10.1145\/775047.775067"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_14_1","DOI":"10.1145\/1076034.1076063"},{"key":"e_1_3_2_1_15_1","volume-title":"The World's First Conference on the Future of Distributed Work (CrowdConf2010)","author":"Kapelner A.","year":"2010","unstructured":"A. Kapelner and D. Chandler. Preventing satisficing in online surveys: A 'kapcha' to ensure higher quality data. In The World's First Conference on the Future of Distributed Work (CrowdConf2010), 2010."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_16_1","DOI":"10.1145\/2009916.2009947"},{"key":"e_1_3_2_1_17_1","first-page":"21","volume-title":"SIGIR Workshop on Crowdsourcing for Search Evaluation","author":"Le J.","year":"2010","unstructured":"J. Le, A. Edmonds, V. Hester, and L. Biewald. Ensuring quality in crowdsourced search relevance evaluation: The effects of training question distribution. In SIGIR Workshop on Crowdsourcing for Search Evaluation, pages 21--26, 2010."},{"key":"e_1_3_2_1_18_1","volume-title":"Overview of the TREC 2011 crowdsourcing track. In Proc. of TREC 2011","author":"Lease M.","year":"2011","unstructured":"M. Lease and G. Kazai. Overview of the TREC 2011 crowdsourcing track. In Proc. of TREC 2011, 2011."},{"key":"e_1_3_2_1_19_1","first-page":"1","volume-title":"Overview of the TREC 2008 legal track. In Proc. 17th Text REtrieval Conference","author":"Oard D. W.","year":"2008","unstructured":"D. W. Oard, B. Hedin, S. Tomlinson, and J. R. Baron. Overview of the TREC 2008 legal track. In Proc. 17th Text REtrieval Conference, pages 1--45, Gaithersburg, Maryland, USA, November 2008. NIST Special Publication 500--277."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_20_1","DOI":"10.1145\/1978942.1979148"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_21_1","DOI":"10.1145\/1935826.1935859"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_22_1","DOI":"10.5555\/1597348.1597412"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_23_1","DOI":"10.1145\/1772690.1772859"},{"key":"e_1_3_2_1_24_1","volume-title":"TREC: Experimentation and Evaluation in Information Retrieval","author":"Voorhees E. M.","year":"2005","unstructured":"E. M. Voorhees and D. K. Harman, editors. TREC: Experimentation and Evaluation in Information Retrieval. MIT Press, 2005."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_25_1","DOI":"10.1145\/2396761.2396781"},{"key":"e_1_3_2_1_26_1","first-page":"17","volume-title":"SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation","author":"Zhu D.","year":"2010","unstructured":"D. Zhu and B. Carterette. An analysis of assessor behavior in crowdsourced preference judgments. In SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation, pages 17--20, 2010."}],"event":{"sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGIR ACM Special Interest Group on Information Retrieval"],"acronym":"CIKM'13","name":"CIKM'13: 22nd ACM International Conference on Information and Knowledge Management","location":"San Francisco California USA"},"container-title":["Proceedings of the 22nd ACM international conference on Information & Knowledge Management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2505515.2505716","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2505515.2505716","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T07:34:09Z","timestamp":1750232049000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2505515.2505716"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,10,27]]},"references-count":26,"alternative-id":["10.1145\/2505515.2505716","10.1145\/2505515"],"URL":"https:\/\/doi.org\/10.1145\/2505515.2505716","relation":{},"subject":[],"published":{"date-parts":[[2013,10,27]]},"assertion":[{"value":"2013-10-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}