{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,7,30]],"date-time":"2025-07-30T15:26:40Z","timestamp":1753889200841,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":20,"publisher":"ACM","license":[{"start":{"date-parts":[[2011,7,24]],"date-time":"2011-07-24T00:00:00Z","timestamp":1311465600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2011,7,24]]},"DOI":"10.1145\/2009916.2010039","type":"proceedings-article","created":{"date-parts":[[2011,7,26]],"date-time":"2011-07-26T14:09:37Z","timestamp":1311689377000},"page":"923-932","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":63,"title":["Repeatable and reliable search system evaluation using crowdsourcing"],"prefix":"10.1145","author":[{"given":"Roi","family":"Blanco","sequence":"first","affiliation":[{"name":"Yahoo! Research, Barcelona, Spain"}]},{"given":"Harry","family":"Halpin","sequence":"additional","affiliation":[{"name":"University of Edinburgh, Edinburgh, United Kingdom"}]},{"given":"Daniel M.","family":"Herzig","sequence":"additional","affiliation":[{"name":"Karlsruhe Institute of Technology, Karlsruhe, Germany"}]},{"given":"Peter","family":"Mika","sequence":"additional","affiliation":[{"name":"Yahoo! Research, Barcelona, Spain"}]},{"given":"Jeffrey","family":"Pound","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, Canada"}]},{"given":"Henry S.","family":"Thompson","sequence":"additional","affiliation":[{"name":"University of Edinburgh, Edinburgh, United Kingdom"}]},{"given":"Thanh","family":"Tran Duc","sequence":"additional","affiliation":[{"name":"Karlsruhe Institute of Technology, Karlsruhe, Germany"}]}],"member":"320","published-online":{"date-parts":[[2011,7,24]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1480506.1480508"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-12275-0_57"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390447"},{"key":"e_1_3_2_1_4_1","volume-title":"Overview of the TREC 2009 Entity Track. In NIST Special Publication: SP 500--278","author":"Balog K.","year":"2009","unstructured":"K. Balog , A. P. de Vries , P. Serdyukov , P. Thomas , and T. Westerveld . Overview of the TREC 2009 Entity Track. In NIST Special Publication: SP 500--278 , 2009 . K. Balog, A. P. de Vries, P. Serdyukov, P. Thomas, and T. Westerveld. Overview of the TREC 2009 Entity Track. In NIST Special Publication: SP 500--278, 2009."},{"key":"e_1_3_2_1_5_1","first-page":"286","volume-title":"Proceedings of the 2009 Conference on Empirical Methods in Natural Language Processing","author":"Callison-Burch C.","year":"2009","unstructured":"C. Callison-Burch . Fast, cheap, and creative : Evaluating translation quality using Amazon's Mechanical Turk . In Proceedings of the 2009 Conference on Empirical Methods in Natural Language Processing , pages 286 -- 295 , Singapore , August 2009 . Association for Computational Linguistics. C. Callison-Burch. Fast, cheap, and creative: Evaluating translation quality using Amazon's Mechanical Turk. In Proceedings of the 2009 Conference on Empirical Methods in Natural Language Processing, pages 286--295, Singapore, August 2009. Association for Computational Linguistics."},{"key":"e_1_3_2_1_6_1","volume-title":"Alias-I, 2008","author":"Carpenter B.","year":"2008","unstructured":"B. Carpenter . Multilevel bayesian models of categorical data annotation. technical report. Technical report , Alias-I, 2008 . http:\/\/lingpipe.files.wordpress.com\/ 2008 \/11\/carp-bayesian-multilevel-annotation.pdf. B. Carpenter. Multilevel bayesian models of categorical data annotation. technical report. Technical report, Alias-I, 2008. http:\/\/lingpipe.files.wordpress.com\/2008\/11\/carp-bayesian-multilevel-annotation.pdf."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/1031171.1031289"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646078"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/775152.775250"},{"key":"e_1_3_2_1_10_1","volume-title":"Proceedings of the WWW Workshop on Linked Data on the Web","author":"Halpin H.","year":"2009","unstructured":"H. Halpin . A query-driven characterization of linked data . In Proceedings of the WWW Workshop on Linked Data on the Web , Madrid, Spain , 2009 . H. Halpin. A query-driven characterization of linked data. In Proceedings of the WWW Workshop on Linked Data on the Web, Madrid, Spain, 2009."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1097-4571(199601)47:1%3C37::AID-ASI4%3E3.3.CO;2-I"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-03761-0_1"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE.2008.4497619"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/1600150.1600175"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1743384.1743478"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1504\/IJMSO.2008.021204"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/1772690.1772769"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.3115\/1220575.1220589"},{"key":"e_1_3_2_1_19_1","volume-title":"SearchWebDB: Data Web Search on a Pay-As-You-Go Integration Infrastructure","author":"Tran T.","year":"2008","unstructured":"T. Tran , H. Wang , and P. Haase . SearchWebDB: Data Web Search on a Pay-As-You-Go Integration Infrastructure , 2008 . T. Tran, H. Wang, and P. Haase. SearchWebDB: Data Web Search on a Pay-As-You-Go Integration Infrastructure, 2008."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.5555\/648264.753539"}],"event":{"name":"SIGIR '11: The 34th International ACM SIGIR conference on research and development in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Beijing China","acronym":"SIGIR '11"},"container-title":["Proceedings of the 34th international ACM SIGIR conference on Research and development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2009916.2010039","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2009916.2010039","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T11:06:28Z","timestamp":1750244788000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2009916.2010039"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2011,7,24]]},"references-count":20,"alternative-id":["10.1145\/2009916.2010039","10.1145\/2009916"],"URL":"https:\/\/doi.org\/10.1145\/2009916.2010039","relation":{},"subject":[],"published":{"date-parts":[[2011,7,24]]},"assertion":[{"value":"2011-07-24","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}