{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,23]],"date-time":"2026-06-23T23:37:26Z","timestamp":1782257846629,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":16,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,5,31]],"date-time":"2020-05-31T00:00:00Z","timestamp":1590883200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,6,11]]},"DOI":"10.1145\/3318464.3383127","type":"proceedings-article","created":{"date-parts":[[2020,5,29]],"date-time":"2020-05-29T17:12:33Z","timestamp":1590772353000},"page":"2623-2627","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":16,"title":["Crowdsourcing Practice for Efficient Data Labeling: Aggregation, Incremental Relabeling, and Pricing"],"prefix":"10.1145","author":[{"given":"Alexey","family":"Drutsa","sequence":"first","affiliation":[{"name":"Yandex, Moscow, Russian Fed."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Valentina","family":"Fedorova","sequence":"additional","affiliation":[{"name":"Yandex, Moscow, Russian Fed."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Dmitry","family":"Ustalov","sequence":"additional","affiliation":[{"name":"Yandex, Saint Petersburg, Russian Fed."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Olga","family":"Megorskaya","sequence":"additional","affiliation":[{"name":"Yandex, Saint Petersburg, Russian Fed."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Evfrosiniya","family":"Zerminova","sequence":"additional","affiliation":[{"name":"Yandex, Moscow, Russian Fed."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Daria","family":"Baidakova","sequence":"additional","affiliation":[{"name":"Yandex, Moscow, Russian Fed."}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2020,5,31]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911514"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1866029.1866078"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10579-012-9180-5"},{"key":"e_1_3_2_1_4_1","volume-title":"Proceedings of the 31st Annual ACM Symposium on User Interface Software and Technology (UIST '18)","author":"Bragg Jonathan","unstructured":"Jonathan Bragg , Mausam, and Daniel S. Weld . 2018. Sprout: Crowd-Powered Task Design for Crowdsourcing . In Proceedings of the 31st Annual ACM Symposium on User Interface Software and Technology (UIST '18) . ACM, Berlin, Germany, 165--176. https:\/\/doi.org\/10.1145\/3242587.3242598 Jonathan Bragg, Mausam, and Daniel S. Weld. 2018. Sprout: Crowd-Powered Task Design for Crowdsourcing. In Proceedings of the 31st Annual ACM Symposium on User Interface Software and Technology (UIST '18). ACM, Berlin, Germany, 165--176. https:\/\/doi.org\/10.1145\/3242587.3242598"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2433396.2433420"},{"key":"e_1_3_2_1_6_1","volume-title":"Proceedings of the 33rd Annual ACM Conference on Human Factors in Computing Systems (CHI '15)","author":"Cheng Justin","unstructured":"Justin Cheng , Jaime Teevan , and Michael S. Bernstein . 2015. Measuring Crowdsourcing Effort with Error-Time Curves . In Proceedings of the 33rd Annual ACM Conference on Human Factors in Computing Systems (CHI '15) . ACM, Seoul, Korea, 1365--1374. https:\/\/doi.org\/10.1145\/2702123.2702145 Justin Cheng, Jaime Teevan, and Michael S. Bernstein. 2015. Measuring Crowdsourcing Effort with Error-Time Curves. In Proceedings of the 33rd Annual ACM Conference on Human Factors in Computing Systems (CHI '15). ACM, Seoul, Korea, 1365--1374. https:\/\/doi.org\/10.1145\/2702123.2702145"},{"key":"e_1_3_2_1_7_1","first-page":"20","article-title":"Maximum Likelihood Estimation of Observer Error-Rates Using the EM Algorithm. Journal of the Royal Statistical Society","volume":"28","author":"Philip Dawid A.","year":"1979","unstructured":"A. Philip Dawid and Allan M. Skene . 1979 . Maximum Likelihood Estimation of Observer Error-Rates Using the EM Algorithm. Journal of the Royal Statistical Society . Series C (Applied Statistics) , Vol. 28 , 1 (1979), 20 -- 28 . https:\/\/doi.org\/10.2307\/2346806 A. Philip Dawid and Allan M. Skene. 1979. Maximum Likelihood Estimation of Observer Error-Rates Using the EM Algorithm. Journal of the Royal Statistical Society. Series C (Applied Statistics), Vol. 28, 1 (1979), 20--28. https:\/\/doi.org\/10.2307\/2346806","journal-title":"Series C (Applied Statistics)"},{"key":"e_1_3_2_1_8_1","volume-title":"Practice of Efficient Data Collection via Crowdsourcing at Large-Scale. Tutorial at the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining (KDD '19)","author":"Drutsa Alexey","year":"2013","unstructured":"Alexey Drutsa , Viktoriya Farafonova , Valentina Fedorova , Olga Megorskaya , Evfrosiniya Zerminova , and Olga Zhilinskaya . 2019. Practice of Efficient Data Collection via Crowdsourcing at Large-Scale. Tutorial at the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining (KDD '19) . Anchorage, AK, USA . Jacob Whitehill, and Javier R. Movellan. 2013 . Exploiting Commonality and Interaction Effects in Crowdsourcing Tasks Using Latent Factor Models. In NIPS '13 Workshop on Crowdsourcing : Theory, Algorithms and Applications . Alexey Drutsa, Viktoriya Farafonova, Valentina Fedorova, Olga Megorskaya, Evfrosiniya Zerminova, and Olga Zhilinskaya. 2019. Practice of Efficient Data Collection via Crowdsourcing at Large-Scale. Tutorial at the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining (KDD '19). Anchorage, AK, USA. Jacob Whitehill, and Javier R. Movellan. 2013. Exploiting Commonality and Interaction Effects in Crowdsourcing Tasks Using Latent Factor Models. In NIPS '13 Workshop on Crowdsourcing: Theory, Algorithms and Applications."},{"key":"e_1_3_2_1_9_1","volume-title":"Proceedings of the First AAAI Conference on Human Computation and Crowdsourcing (HCOMP","author":"Sheshadri Aashish","year":"2013","unstructured":"Aashish Sheshadri and Matthew Lease . 2013 . SQUARE: A Benchmark for Research on Computing Crowd Consensus . In Proceedings of the First AAAI Conference on Human Computation and Crowdsourcing (HCOMP 2013). Association for the Advancement of Artificial Intelligence, Palm Springs, CA, USA, 156--164. Aashish Sheshadri and Matthew Lease. 2013. SQUARE: A Benchmark for Research on Computing Crowd Consensus. In Proceedings of the First AAAI Conference on Human Computation and Crowdsourcing (HCOMP 2013). Association for the Advancement of Artificial Intelligence, Palm Springs, CA, USA, 156--164."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.3115\/1613715.1613751"},{"key":"e_1_3_2_1_11_1","unstructured":"Jing Wang Panagiotis G. Ipeirotis and Foster Provost. 2013. Quality-Based Pricing for Crowdsourced Workers. (2013). NYU Working Paper No. 2451\/31833.  Jing Wang Panagiotis G. Ipeirotis and Foster Provost. 2013. Quality-Based Pricing for Crowdsourced Workers. (2013). NYU Working Paper No. 2451\/31833."},{"key":"e_1_3_2_1_12_1","volume-title":"Belongie","author":"Welinder Peter","year":"2010","unstructured":"Peter Welinder , Steve Branson , Pietro Perona , and Serge J . Belongie . 2010 . The Multidimensional Wisdom of Crowds. In Advances in Neural Information Processing Systems 21 (NIPS 2010). Curran Associates, Inc., Vancouver, BC, Canada , 2424--2432. Peter Welinder, Steve Branson, Pietro Perona, and Serge J. Belongie. 2010. The Multidimensional Wisdom of Crowds. In Advances in Neural Information Processing Systems 21 (NIPS 2010). Curran Associates, Inc., Vancouver, BC, Canada, 2424--2432."},{"key":"e_1_3_2_1_13_1","volume-title":"Ruvolo","author":"Whitehill Jacob","year":"2009","unstructured":"Jacob Whitehill , Ting-fan Wu, Jacob Bergsma , Javier R. Movellan , and Paul L . Ruvolo . 2009 . Whose Vote Should Count More: Optimal Integration of Labels from Labelers of Unknown Expertise. In Advances in Neural Information Processing Systems 22 (NIPS 2009). Curran Associates, Inc., Vancouver, BC, Canada , 2035--2043. Jacob Whitehill, Ting-fan Wu, Jacob Bergsma, Javier R. Movellan, and Paul L. Ruvolo. 2009. Whose Vote Should Count More: Optimal Integration of Labels from Labelers of Unknown Expertise. In Advances in Neural Information Processing Systems 22 (NIPS 2009). Curran Associates, Inc., Vancouver, BC, Canada, 2035--2043."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.5555\/2891460.2891626"},{"key":"e_1_3_2_1_15_1","volume-title":"Security, Risk and Trust and 2011 IEEE Third International Conference on Social Computing","author":"Zhao Liyue","year":"2011","unstructured":"Liyue Zhao , Gita Sukthankar , and Rahul Sukthankar . 2011. Incremental Relabeling for Active Learning with Noisy Crowdsourced Annotations. In 2011 IEEE Third International Conference on Privacy , Security, Risk and Trust and 2011 IEEE Third International Conference on Social Computing . IEEE , Boston, MA, USA , 728--733. https:\/\/doi.org\/10.1109\/passat\/socialcom. 2011 .193 Liyue Zhao, Gita Sukthankar, and Rahul Sukthankar. 2011. Incremental Relabeling for Active Learning with Noisy Crowdsourced Annotations. In 2011 IEEE Third International Conference on Privacy, Security, Risk and Trust and 2011 IEEE Third International Conference on Social Computing. IEEE, Boston, MA, USA, 728--733. https:\/\/doi.org\/10.1109\/passat\/socialcom.2011.193"},{"key":"e_1_3_2_1_16_1","volume-title":"Shah","author":"Zhou Dengyong","year":"2015","unstructured":"Dengyong Zhou , Qiang Liu , John C. Platt , Christopher Meek , and Nihar B . Shah . 2015 . Regularized Minimax Conditional Entropy for Crowdsourcing . 1503.07240 Dengyong Zhou, Qiang Liu, John C. Platt, Christopher Meek, and Nihar B. Shah. 2015. Regularized Minimax Conditional Entropy for Crowdsourcing. 1503.07240"}],"event":{"name":"SIGMOD\/PODS '20: International Conference on Management of Data","location":"Portland OR USA","acronym":"SIGMOD\/PODS '20","sponsor":["SIGMOD ACM Special Interest Group on Management of Data"]},"container-title":["Proceedings of the 2020 ACM SIGMOD International Conference on Management of Data"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3318464.3383127","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3318464.3383127","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:38:23Z","timestamp":1750199903000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3318464.3383127"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,5,31]]},"references-count":16,"alternative-id":["10.1145\/3318464.3383127","10.1145\/3318464"],"URL":"https:\/\/doi.org\/10.1145\/3318464.3383127","relation":{},"subject":[],"published":{"date-parts":[[2020,5,31]]},"assertion":[{"value":"2020-05-31","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}