{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,7,4]],"date-time":"2025-07-04T06:15:00Z","timestamp":1751609700796,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":29,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,1,20]],"date-time":"2020-01-20T00:00:00Z","timestamp":1579478400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,1,20]]},"DOI":"10.1145\/3336191.3371875","type":"proceedings-article","created":{"date-parts":[[2020,1,22]],"date-time":"2020-01-22T19:08:16Z","timestamp":1579720096000},"page":"873-876","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":8,"title":["Practice of Efficient Data Collection via Crowdsourcing"],"prefix":"10.1145","author":[{"given":"Alexey","family":"Drutsa","sequence":"first","affiliation":[{"name":"Yandex, Moscow, Russian Fed."}]},{"given":"Valentina","family":"Fedorova","sequence":"additional","affiliation":[{"name":"Yandex, Moscow, Russian Fed."}]},{"given":"Dmitry","family":"Ustalov","sequence":"additional","affiliation":[{"name":"Yandex, Saint Petersburg, Russian Fed."}]},{"given":"Olga","family":"Megorskaya","sequence":"additional","affiliation":[{"name":"Yandex, Saint Petersburg, Russian Fed."}]},{"given":"Evfrosiniya","family":"Zerminova","sequence":"additional","affiliation":[{"name":"Yandex, Moscow, Russian Fed."}]},{"given":"Daria","family":"Baidakova","sequence":"additional","affiliation":[{"name":"Yandex, Moscow, Russian Fed."}]}],"member":"320","published-online":{"date-parts":[[2020,1,22]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911514"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/2433396.2433420"},{"volume-title":"Proceedings of the 33rd Annual ACM Conference on Human Factors in Computing Systems (CHI '15)","author":"Cheng Justin","key":"e_1_3_2_1_3_1"},{"key":"e_1_3_2_1_4_1","first-page":"20","article-title":"Maximum Likelihood Estimation of Observer Error-Rates Using the EM Algorithm. Journal of the Royal Statistical Society","volume":"28","author":"Philip Dawid A.","year":"1979","journal-title":"Series C (Applied Statistics)"},{"volume-title":"Proceedings of the Second AAAI Conference on Human Computation and Crowdsourcing (HCOMP","year":"2014","author":"Difallah Djellel Eddine","key":"e_1_3_2_1_5_1"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"Alexey Drutsa Viktoriya Farafonova Valentina Fedorova Olga Megorskaya Evfrosiniya Zerminova and Olga Zhilinskaya. 2019. Practice of Efficient Data Collection via Crowdsourcing at Large-Scale. Tutorial at KDD '19.  Alexey Drutsa Viktoriya Farafonova Valentina Fedorova Olga Megorskaya Evfrosiniya Zerminova and Olga Zhilinskaya. 2019. Practice of Efficient Data Collection via Crowdsourcing at Large-Scale. Tutorial at KDD '19.","DOI":"10.1145\/3336191.3371875"},{"key":"e_1_3_2_1_7_1","unstructured":"Seyda Ertekin Haym Hirsh and Cynthia Rudin. 2012. Learning to Predict the Wisdom of Crowds. arXiv:1204.3611  Seyda Ertekin Haym Hirsh and Cynthia Rudin. 2012. Learning to Predict the Wisdom of Crowds. arXiv:1204.3611"},{"volume-title":"Human Computation and Crowdsourcing: Works in Progress and Demonstration Abstracts (HCOMP","year":"2013","author":"Faridani Siamak","key":"e_1_3_2_1_8_1"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2736277.2741102"},{"key":"e_1_3_2_1_10_1","unstructured":"Hideaki Imamura Issei Sato and Masashi Sugiyama. 2018. Analysis of Minimax Error Rate for Crowdsourcing and Its Application to Worker Clustering Model. arXiv:1802.04551  Hideaki Imamura Issei Sato and Masashi Sugiyama. 2018. Analysis of Minimax Error Rate for Crowdsourcing and Its Application to Worker Clustering Model. arXiv:1802.04551"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-013-0306-1"},{"volume-title":"Proceedings of the Fifth Conference on Human Computation and Crowdsourcing (HCOMP","year":"2017","author":"Jin Yuan","key":"e_1_3_2_1_12_1"},{"volume-title":"Proceedings of the Fifteenth International Conference on Artificial Intelligence and Statistics (AISTATS","year":"2012","author":"Kim Hyun-Chul","key":"e_1_3_2_1_13_1"},{"volume-title":"Proceedings of the Second AAAI Conference on Human Computation and Crowdsourcing (HCOMP","year":"2014","author":"Lin Christopher H.","key":"e_1_3_2_1_14_1"},{"volume-title":"Proceedings of the 29th International Conference on Machine Learning (ICML-12)","year":"2012","author":"Liu Chao","key":"e_1_3_2_1_15_1"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.5555\/1756006.1859894"},{"volume-title":"Exploiting Commonality and Interaction Effects in Crowdsourcing Tasks Using Latent Factor Models. In NIPS '13 Workshop on Crowdsourcing: Theory, Algorithms and Applications.","year":"2013","author":"Ruvolo Paul","key":"e_1_3_2_1_17_1"},{"volume-title":"Proceedings of the 33rd International Conference on International Conference on Machine Learning (ICML16)","author":"Nihar","key":"e_1_3_2_1_18_1"},{"volume-title":"Proceedings of the 32Nd International Conference on International Conference on Machine Learning (ICML-15)","year":"2015","author":"Shah Nihar B.","key":"e_1_3_2_1_19_1"},{"volume-title":"Proceedings of the First AAAI Conference on Human Computation and Crowdsourcing (HCOMP","year":"2013","author":"Sheshadri Aashish","key":"e_1_3_2_1_20_1"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.3115\/1613715.1613751"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2566486.2567989"},{"volume-title":"Proceedings of the ACM SIGIR 2011 Workshop on Crowdsourcing for Information Retrieval (CIR","year":"2011","author":"Vuurens Jeroen","key":"e_1_3_2_1_23_1"},{"key":"e_1_3_2_1_24_1","unstructured":"Jing Wang Panagiotis G. Ipeirotis and Foster Provost. 2013. Quality-Based Pricing for Crowdsourced Workers. (2013). NYU Working Paper No. 2451\/31833.  Jing Wang Panagiotis G. Ipeirotis and Foster Provost. 2013. Quality-Based Pricing for Crowdsourced Workers. (2013). NYU Working Paper No. 2451\/31833."},{"key":"e_1_3_2_1_25_1","unstructured":"Peter Welinder Steve Branson Pietro Perona and Serge J. Belongie. 2010. The Multidimensional Wisdom of Crowds. In Advances in Neural Information Processing Systems 21 (NIPS 2010). Curran Associates Inc. Vancouver BC Canada 2424--2432.  Peter Welinder Steve Branson Pietro Perona and Serge J. Belongie. 2010. The Multidimensional Wisdom of Crowds. In Advances in Neural Information Processing Systems 21 (NIPS 2010). Curran Associates Inc. Vancouver BC Canada 2424--2432."},{"key":"e_1_3_2_1_26_1","unstructured":"Jacob Whitehill Ting-fan Wu Jacob Bergsma Javier R. Movellan and Paul L. Ruvolo. 2009. Whose Vote Should Count More: Optimal Integration of Labels from Labelers of Unknown Expertise. In Advances in Neural Information Processing Systems 22 (NIPS 2009). Curran Associates Inc. Vancouver BC Canada 2035-- 2043.  Jacob Whitehill Ting-fan Wu Jacob Bergsma Javier R. Movellan and Paul L. Ruvolo. 2009. Whose Vote Should Count More: Optimal Integration of Labels from Labelers of Unknown Expertise. In Advances in Neural Information Processing Systems 22 (NIPS 2009). Curran Associates Inc. Vancouver BC Canada 2035-- 2043."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.5555\/2891460.2891626"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"crossref","unstructured":"Liyue Zhao Gita Sukthankar and Rahul Sukthankar. 2011. Incremental Relabeling for Active Learning with Noisy Crowdsourced Annotations. In 2011 IEEE Third International Conference on Privacy Security Risk and Trust and 2011 IEEE Third International Conference on Social Computing. IEEE Boston MA USA 728--733.  Liyue Zhao Gita Sukthankar and Rahul Sukthankar. 2011. Incremental Relabeling for Active Learning with Noisy Crowdsourced Annotations. In 2011 IEEE Third International Conference on Privacy Security Risk and Trust and 2011 IEEE Third International Conference on Social Computing. IEEE Boston MA USA 728--733.","DOI":"10.1109\/PASSAT\/SocialCom.2011.193"},{"key":"e_1_3_2_1_29_1","unstructured":"Dengyong Zhou Qiang Liu John C. Platt Christopher Meek and Nihar B. Shah. 2015. Regularized Minimax Conditional Entropy for Crowdsourcing. arXiv:1503.07240  Dengyong Zhou Qiang Liu John C. Platt Christopher Meek and Nihar B. Shah. 2015. Regularized Minimax Conditional Entropy for Crowdsourcing. arXiv:1503.07240"}],"event":{"name":"WSDM '20: The Thirteenth ACM International Conference on Web Search and Data Mining","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data","SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Houston TX USA","acronym":"WSDM '20"},"container-title":["Proceedings of the 13th International Conference on Web Search and Data Mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3336191.3371875","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3336191.3371875","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T23:23:15Z","timestamp":1750202595000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3336191.3371875"}},"subtitle":["Aggregation, Incremental Relabelling, and Pricing"],"short-title":[],"issued":{"date-parts":[[2020,1,20]]},"references-count":29,"alternative-id":["10.1145\/3336191.3371875","10.1145\/3336191"],"URL":"https:\/\/doi.org\/10.1145\/3336191.3371875","relation":{},"subject":[],"published":{"date-parts":[[2020,1,20]]},"assertion":[{"value":"2020-01-22","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}