{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,12]],"date-time":"2026-05-12T21:11:29Z","timestamp":1778620289600,"version":"3.51.4"},"reference-count":42,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2020,10,2]],"date-time":"2020-10-02T00:00:00Z","timestamp":1601596800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,10,2]],"date-time":"2020-10-02T00:00:00Z","timestamp":1601596800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Front. Comput. Sci."],"published-print":{"date-parts":[[2021,8]]},"DOI":"10.1007\/s11704-020-9364-x","type":"journal-article","created":{"date-parts":[[2020,10,2]],"date-time":"2020-10-02T07:02:58Z","timestamp":1601622178000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":11,"title":["Find truth in the hands of the few: acquiring specific knowledge with crowdsourcing"],"prefix":"10.1007","volume":"15","author":[{"given":"Tao","family":"Han","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hailong","family":"Sun","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yangqiu","family":"Song","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yili","family":"Fang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xudong","family":"Liu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2020,10,2]]},"reference":[{"issue":"6","key":"9364_CR1","first-page":"1","volume":"14","author":"J Howe","year":"2006","unstructured":"Howe J. The rise of crowdsourcing. Wired Magazine, 2006, 14(6): 1\u20134","journal-title":"Wired Magazine"},{"key":"9364_CR2","doi-asserted-by":"crossref","unstructured":"Wang J, Li G, Kraska T, Franklin M J, Feng J. Leveraging transitive relations for crowdsourced joins. In: Proceedings of ACM Conference on Management of Data. 2013, 229\u2013240","DOI":"10.1145\/2463676.2465280"},{"issue":"1\u20133","key":"9364_CR3","doi-asserted-by":"publisher","first-page":"157","DOI":"10.1007\/s11263-007-0090-8","volume":"77","author":"B C Russell","year":"2008","unstructured":"Russell B C, Torralba A, Murphy K P, Freeman W T. Labelme: a database and Web-based tool for image annotation. International Journal of Computer Vision, 2008, 77(1\u20133): 157\u2013173","journal-title":"International Journal of Computer Vision"},{"issue":"2","key":"9364_CR4","doi-asserted-by":"publisher","first-page":"700","DOI":"10.1109\/TCE.2012.6227479","volume":"58","author":"K Hwang","year":"2012","unstructured":"Hwang K, Lee S Y. Environmental audio scene and activity recognition through mobile-based crowdsourcing. IEEE Transactions on Consumer Electronics, 2012, 58(2): 700\u2013705","journal-title":"IEEE Transactions on Consumer Electronics"},{"issue":"1","key":"9364_CR5","doi-asserted-by":"publisher","first-page":"184","DOI":"10.1007\/s11263-012-0564-1","volume":"101","author":"C Vondrick","year":"2013","unstructured":"Vondrick C, Patterson D, Ramanan D. Efficiently scaling up crowdsourced video annotation. International Journal of Computer Vision, 2013, 101(1): 184\u2013204","journal-title":"International Journal of Computer Vision"},{"key":"9364_CR6","doi-asserted-by":"crossref","unstructured":"Waggoner B, Chen Y. Output agreement mechanisms and common knowledge. In: Proceedings of the 2nd AAAI Conference on Human Computation and Crowdsourcing. 2014","DOI":"10.1609\/hcomp.v2i1.13151"},{"key":"9364_CR7","doi-asserted-by":"crossref","unstructured":"Ordonez V, Deng J, Choi Y, Berg A C, Berg T. From large scale image categorization to entry-level categories. In: Proceedings of IEEE International Conference on Computer Vision. 2013, 2768\u20132775","DOI":"10.1109\/ICCV.2013.344"},{"key":"9364_CR8","doi-asserted-by":"crossref","unstructured":"Feng S, Ravi S, Kumar R, Kuznetsova P, Liu W, Berg A C, Berg T L, Choi Y. Refer-to-as relations as semantic knowledge. In: Proceedings of International Conference on Automated Planning and Scheduling. 2015","DOI":"10.1609\/aaai.v29i1.9489"},{"issue":"1","key":"9364_CR9","doi-asserted-by":"publisher","first-page":"20","DOI":"10.2307\/2346806","volume":"28","author":"A P Dawid","year":"1979","unstructured":"Dawid A P, Skene A M. Maximum likelihood estimation of observer error-rates using the em algorithm. Applied Statistics, 1979, 28(1): 20\u201328","journal-title":"Applied Statistics"},{"key":"9364_CR10","unstructured":"Whitehill J, Wu T F, Bergsma J, Movellan J R, Ruvolo P L. Whose vote should count more: optimal integration of labels from labelers of unknown expertise. In: Proceedings of Annual Conference on Neural Information Processing Systems. 2009, 2035\u20132043"},{"key":"9364_CR11","doi-asserted-by":"crossref","unstructured":"Salek M, Bachrach Y, Key P. Hotspotting-a probabilistic graphical model for image object localization through crowdsourcing. In: Proceedings of International Conference on Automated Planning and Scheduling. 2013","DOI":"10.1609\/aaai.v27i1.8465"},{"key":"9364_CR12","unstructured":"Bachrach Y, Minka T, Guiver J, Graepel T. How to grade a test without knowing the answers\u2014a bayesian graphical model for adaptive crowdsourcing and aptitude testing. In: Proceedings of the 29th International Conference on Machine Learning. 2012, 819\u2013826"},{"issue":"43","key":"9364_CR13","first-page":"1297","volume":"11","author":"V C Raykar","year":"2010","unstructured":"Raykar V C, Yu S, Zhao L H, Valadez G H, Florin C, Bogoni L, Moy L. Learning from crowds. Journal of Machine Learning Research, 2010, 11(43): 1297\u20131322","journal-title":"Journal of Machine Learning Research"},{"key":"9364_CR14","doi-asserted-by":"crossref","unstructured":"Demartini G, Difallah D E, Cudr\u00e9-Mauroux P. Zencrowd: leveraging probabilistic reasoning and crowdsourcing techniques for large-scale entity linking. In: Proceedings of the 21st International Conference on World Wide Web. 2012, 469\u2013478","DOI":"10.1145\/2187836.2187900"},{"key":"9364_CR15","unstructured":"Zhou D, Basu S, Mao Y, Platt J C. Learning from the wisdom of crowds by minimax entropy. In: Proceedings of Annual Conference on Neural Information Processing Systems. 2012, 2195\u20132203"},{"key":"9364_CR16","unstructured":"Han T, Sun H, Song Y, Fang Y, Liu X. Incorporating external knowledge into crowd intelligence for more specific knowledge acquisition. In: Proceedings of International Joint Conference on Artificial Intelligence. 2016, 1541\u20131547"},{"key":"9364_CR17","doi-asserted-by":"crossref","unstructured":"Chilton L B, Little G, Edge D, Weld D S, Landay J A. Cascade: crowdsourcing taxonomy creation. In: Proceedings of SIGCHI Conference on Human Factors in Computing Systems. 2013, 1999\u20132008","DOI":"10.1145\/2470654.2466265"},{"key":"9364_CR18","doi-asserted-by":"crossref","unstructured":"Bragg J, Weld D S. Crowdsourcing multi-label classification for taxonomy creation. In: Proceedings of the 1st AAAI Conference on Human Computation and Crowdsourcing. 2013","DOI":"10.1609\/hcomp.v1i1.13091"},{"key":"9364_CR19","unstructured":"Sun Y, Singla A, Fox D, Krause A. Building hierarchies of concepts via crowdsourcing. In: Proceedings of International Joint Conference on Artificial Intelligence. 2015, 844\u2013851"},{"key":"9364_CR20","doi-asserted-by":"crossref","unstructured":"Fellbaum C. WordNet: An Electronic Lexical Database. MIT Press, 1998","DOI":"10.7551\/mitpress\/7287.001.0001"},{"key":"9364_CR21","unstructured":"Lenat D B, Guha R V. Building Large Knowledge-Based Systems: Representation and Inference in the Cyc Project. Addison-Wesley, 1989"},{"key":"9364_CR22","unstructured":"Speer R, Havasi C. Representing general relational knowledge in conceptnet 5. In: Proceedings of Language Resources and Evaluation Conference. 2012, 3679\u20133686"},{"key":"9364_CR23","doi-asserted-by":"crossref","unstructured":"Wu W, Li H, Wang H, Zhu K Q. Probase: a probabilistic taxonomy for text understanding. In: Proceedings of ACM Conference on Management of Data. 2012, 481\u2013492","DOI":"10.1145\/2213836.2213891"},{"issue":"7638","key":"9364_CR24","doi-asserted-by":"publisher","first-page":"532","DOI":"10.1038\/nature21054","volume":"541","author":"D Prelec","year":"2017","unstructured":"Prelec D, Seung H S, McCoy J. A solution to the single-question crowd wisdom problem. Nature, 2017, 541(7638): 532\u2013535","journal-title":"Nature"},{"key":"9364_CR25","doi-asserted-by":"crossref","unstructured":"Divvala S K, Farhadi A, Guestrin C. Learning everything about anything: webly-supervised visual concept learning. In: Proceedings of IEEE Conference on Computer Vision and Pattern Recognition. 2014, 3270\u20133277","DOI":"10.1109\/CVPR.2014.412"},{"key":"9364_CR26","doi-asserted-by":"crossref","unstructured":"Sheng V S, Provost F, Ipeirotis P G. Get another label? improving data quality and data mining using multiple, noisy labelers. In: Proceedings of the 14th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining. 2008, 614\u2013622","DOI":"10.1145\/1401890.1401965"},{"key":"9364_CR27","doi-asserted-by":"crossref","unstructured":"Ipeirotis P G, Provost F, Wang J. Quality management on amazon mechanical turk. In: Proceedings of the ACM SIGKDD Workshop on Human Computation. 2010, 64\u201367","DOI":"10.1145\/1837885.1837906"},{"key":"9364_CR28","doi-asserted-by":"crossref","unstructured":"Han T, Sun H, Song Y, Wang Z, Liu X. Budgeted task scheduling for crowdsourced knowledge acquisition. In: Proceedings of the 2017 ACM on Conference on Information and Knowledge Management. 2017, 1059\u20131068","DOI":"10.1145\/3132847.3133002"},{"key":"9364_CR29","doi-asserted-by":"crossref","unstructured":"Callison-Burch C. Fast, cheap, and creative: evaluating translation quality using amazon\u2019s mechanical turk. In: Proceedings of the 2009 Conference on Empirical Methods in Natural Language Processing. 2009, 286\u2013295","DOI":"10.3115\/1699510.1699548"},{"key":"9364_CR30","doi-asserted-by":"crossref","unstructured":"Hu C, Bederson B B, Resnik P. Translation by iterative collaboration between monolingual users. In: Proceedings of Graphics Interface 2010. 2010, 39\u201346","DOI":"10.1145\/1837885.1837902"},{"key":"9364_CR31","unstructured":"Ambati V, Vogel S, Carbonell J. Active learning and crowd-sourcing for machine translation. In: Proceedings of the 7th International Conference on Language Resources and Evaluation. 2010"},{"issue":"10","key":"9364_CR32","doi-asserted-by":"publisher","first-page":"881","DOI":"10.14778\/2732951.2732962","volume":"7","author":"X L Dong","year":"2014","unstructured":"Dong X L, Gabrilovich E, Heitz G, Horn W, Murphy K, Sun S, Zhang W. From data fusion to knowledge fusion. Proceedings of the VLDB Endowment, 2014, 7(10): 881\u2013892","journal-title":"Proceedings of the VLDB Endowment"},{"key":"9364_CR33","doi-asserted-by":"crossref","unstructured":"Ma F, Li Y, Li Q, Qiu M, Gao J, Zhi S, Su L, Zhao B, Ji H, Han J. Faitcrowd: fine grained truth discovery for crowdsourced data aggregation. In: Proceedings of the 21st ACM SIGKDD International Conference on Knowledge Discovery and Data Mining. 2015, 745\u2013754","DOI":"10.1145\/2783258.2783314"},{"key":"9364_CR34","doi-asserted-by":"crossref","unstructured":"Fang Y, Sun H, Chen P, Huai J. On the cost complexity of crowdsourcing. In: Proceedings of International Joint Conference on Artificial Intelligence. 2018, 1531\u20131537","DOI":"10.24963\/ijcai.2018\/212"},{"issue":"6","key":"9364_CR35","doi-asserted-by":"publisher","first-page":"e167","DOI":"10.2196\/jmir.2338","volume":"14","author":"M A Luengo-Oroz","year":"2012","unstructured":"Luengo-Oroz M A, Arranz A, Frean J. Crowdsourcing malaria parasite quantification: an online game for analyzing images of infected thick blood smears. Journal of Medical Internet Research, 2012, 14(6): e167","journal-title":"Journal of Medical Internet Research"},{"issue":"1","key":"9364_CR36","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1115\/1.3662552","volume":"82","author":"R E Kalman","year":"1960","unstructured":"Kalman R E. A new approach to linear filtering and prediction problems. Journal of Basic Engineering, 1960, 82(1): 35\u201345","journal-title":"Journal of Basic Engineering"},{"issue":"5","key":"9364_CR37","doi-asserted-by":"publisher","first-page":"1389","DOI":"10.1109\/JIOT.2017.2673958","volume":"4","author":"H Sun","year":"2017","unstructured":"Sun H, Hu K, Fang Y, Song Y. Adaptive result inference for collecting quantitative data with crowdsourcing. IEEE Internet of Things Journal, 2017, 4(5): 1389\u20131398","journal-title":"IEEE Internet of Things Journal"},{"key":"9364_CR38","doi-asserted-by":"publisher","first-page":"52","DOI":"10.1016\/j.artint.2013.06.002","volume":"202","author":"P Dai","year":"2013","unstructured":"Dai P, Lin C H, Weld D S. Pomdp-based control of workflows for crowdsourcing. Artificial Intelligence, 2013, 202: 52\u201385","journal-title":"Artificial Intelligence"},{"key":"9364_CR39","doi-asserted-by":"crossref","unstructured":"Dai P, Weld D S. Artificial intelligence for artificial artificial intelligence. In: Proceedings of the 25th AAAI Conference on Artificial Intelligence. 2011","DOI":"10.1609\/aaai.v25i1.8096"},{"key":"9364_CR40","doi-asserted-by":"publisher","first-page":"346","DOI":"10.1016\/j.ins.2018.05.050","volume":"460","author":"Y Fang","year":"2018","unstructured":"Fang Y, Sun H, Li G, Zhang R, Huai J. Context-aware result inference in crowdsourcing. Information Sciences, 2018, 460: 346\u2013363","journal-title":"Information Sciences"},{"key":"9364_CR41","doi-asserted-by":"publisher","first-page":"155","DOI":"10.1016\/j.eswa.2016.04.009","volume":"58","author":"N Otani","year":"2016","unstructured":"Otani N, Baba Y, Kashima H. Quality control of crowdsourced classification using hierarchical class structures. Expert Systems with Applications, 2016, 58: 155\u2013163","journal-title":"Expert Systems with Applications"},{"key":"9364_CR42","doi-asserted-by":"crossref","unstructured":"Deng J, Dong W, Socher R, Li L J, Li K, Fei-Fei L. Imagenet: a large-scale hierarchical image database. In: Proceedings of IEEE Conference on Computer Vision and Pattern Recognition. 2009, 248\u2013255","DOI":"10.1109\/CVPR.2009.5206848"}],"container-title":["Frontiers of Computer Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11704-020-9364-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11704-020-9364-x\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11704-020-9364-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,11,21]],"date-time":"2022-11-21T22:58:26Z","timestamp":1669071506000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11704-020-9364-x"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,10,2]]},"references-count":42,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2021,8]]}},"alternative-id":["9364"],"URL":"https:\/\/doi.org\/10.1007\/s11704-020-9364-x","relation":{},"ISSN":["2095-2228","2095-2236"],"issn-type":[{"value":"2095-2228","type":"print"},{"value":"2095-2236","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,10,2]]},"assertion":[{"value":"24 September 2019","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"3 April 2020","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"2 October 2020","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"154315"}}