{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T04:10:56Z","timestamp":1750306256045,"version":"3.41.0"},"publisher-location":"Republic and Canton of Geneva, Switzerland","reference-count":20,"publisher":"International World Wide Web Conferences Steering Committee","license":[{"start":{"date-parts":[[2017,4,3]],"date-time":"2017-04-03T00:00:00Z","timestamp":1491177600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2017,4,3]]},"DOI":"10.1145\/3038912.3052570","type":"proceedings-article","created":{"date-parts":[[2017,4,6]],"date-time":"2017-04-06T13:30:38Z","timestamp":1491485438000},"page":"529-537","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Drawing Sound Conclusions from Noisy Judgments"],"prefix":"10.1145","author":[{"given":"David","family":"Goldberg","sequence":"first","affiliation":[{"name":"eBay, San Jose, CA, USA"}]},{"given":"Andrew","family":"Trotman","sequence":"additional","affiliation":[{"name":"University of Otago, Dunedin, New Zealand"}]},{"given":"Xiao","family":"Wang","sequence":"additional","affiliation":[{"name":"eBay, San Jose, CA, USA"}]},{"given":"Wei","family":"Min","sequence":"additional","affiliation":[{"name":"CreditX, Shanghai, China"}]},{"given":"Zongru","family":"Wan","sequence":"additional","affiliation":[{"name":"Evolution Labs, Shanghia, China"}]}],"member":"320","published-online":{"date-parts":[[2017,4,3]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1480506.1480508"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390447"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148262"},{"key":"e_1_3_2_1_4_1","series-title":"Series C (Applied Statistics), 28(1):20--28","volume-title":"Maximum likelihood estimation of observer error-rates using the EM algorithm. Journal of the Royal Statistical Society","author":"Dawid A. P.","year":"1979","unstructured":"A. P. Dawid and A. M. Skene. Maximum likelihood estimation of observer error-rates using the EM algorithm. Journal of the Royal Statistical Society. Series C (Applied Statistics), 28(1):20--28, 1979."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1008992.1009079"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2487575.2487595"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1185877.1185883"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00185"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416951"},{"key":"e_1_3_2_1_12_1","first-page":"60","volume-title":"ADCS 2010","author":"Sanderson M.","year":"2010","unstructured":"M. Sanderson, F. Scholer, and A. Turpin. Relatively relevant: Assessor shift in document judgements. In ADCS 2010, pages 60--67, 2010."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010057"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/1321440.1321528"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.5555\/1613715.1613751"},{"key":"e_1_3_2_1_16_1","first-page":"36","volume-title":"SIGIR 2011 Workshop on Crowdsourcing for Information Retrieval","author":"Tang W.","year":"2011","unstructured":"W. Tang and M. Lease. Semi-supervised consensus labeling for crowdsourcing. In SIGIR 2011 Workshop on Crowdsourcing for Information Retrieval, pages 36--41, 2011."},{"key":"e_1_3_2_1_17_1","first-page":"21","volume-title":"ACM SIGIR Workshop on Crowdsourcing for Information Retrieval","author":"Vuurens J.","year":"2011","unstructured":"J. Vuurens, A. de Vries, and C. Eickhoff. How Much Spam Can You Take? An Analysis of Crowdsourcing Results to Increase Accuracy. In ACM SIGIR Workshop on Crowdsourcing for Information Retrieval, pages 21--26, 2011."},{"key":"e_1_3_2_1_18_1","first-page":"1","volume-title":"Liu. A Theoretical Analysis of NDCG Ranking Measures. In 26th Annual Conference on Learning Theory","author":"Wang Y.","year":"2013","unstructured":"Y. Wang, L. Wang, Y. Li, D. He, W. Chen, and T.-Y. Liu. A Theoretical Analysis of NDCG Ranking Measures. In 26th Annual Conference on Learning Theory, pages 1--30, 2013."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/1852102.1852106"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.14778\/2168651.2168656"}],"event":{"name":"WWW '17: 26th International World Wide Web Conference","sponsor":["IW3C2 International World Wide Web Conference Committee","SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"],"location":"Perth Australia","acronym":"WWW '17"},"container-title":["Proceedings of the 26th International Conference on World Wide Web"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3038912.3052570","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3038912.3052570","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:24:06Z","timestamp":1750220646000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3038912.3052570"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,4,3]]},"references-count":20,"alternative-id":["10.1145\/3038912.3052570","10.5555\/3038912"],"URL":"https:\/\/doi.org\/10.1145\/3038912.3052570","relation":{},"subject":[],"published":{"date-parts":[[2017,4,3]]},"assertion":[{"value":"2017-04-03","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}