{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T05:25:29Z","timestamp":1755926729342,"version":"3.28.0"},"reference-count":43,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2022,12,27]],"date-time":"2022-12-27T00:00:00Z","timestamp":1672099200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,12,27]],"date-time":"2022-12-27T00:00:00Z","timestamp":1672099200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Front. Comput. Sci."],"published-print":{"date-parts":[[2023,10]]},"DOI":"10.1007\/s11704-022-2245-8","type":"journal-article","created":{"date-parts":[[2022,12,27]],"date-time":"2022-12-27T07:02:42Z","timestamp":1672124562000},"update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Classification-oriented dawid skene model for transferring intelligence from crowds to machines"],"prefix":"10.1007","volume":"17","author":[{"given":"Jiaran","family":"Li","sequence":"first","affiliation":[]},{"given":"Richong","family":"Zhang","sequence":"additional","affiliation":[]},{"given":"Samuel","family":"Mensah","sequence":"additional","affiliation":[]},{"given":"Wenyi","family":"Qin","sequence":"additional","affiliation":[]},{"given":"Chunming","family":"Hu","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2022,12,27]]},"reference":[{"key":"2245_CR1","unstructured":"Su H, Deng J, Li F F. Crowdsourcing annotations for visual object detection. In: Proceedings of 2012 AAAI Workshop on Human Computation. 2012, 40\u201346"},{"key":"2245_CR2","unstructured":"Welinder P, Branson S, Belongie S, Perona P. The multidimensional wisdom of crowds. In: Proceedings of the 23rd International Conference on Neural Information Processing Systems. 2010, 2424\u20132432"},{"key":"2245_CR3","doi-asserted-by":"crossref","unstructured":"Little G, Chilton L B, Goldman M, Miller R C. TurKit: human computation algorithms on mechanical Turk. In: Proceedings of the 23rd Annual ACM Symposium on User Interface Software and Technology. 2010, 57\u201366","DOI":"10.1145\/1866029.1866040"},{"key":"2245_CR4","doi-asserted-by":"crossref","unstructured":"Snow R, O\u2019Connor B, Jurafsky D, Ng A Y. Cheap and fast\u2014but is it good?: evaluating non-expert annotations for natural language tasks. In: Proceedings of the Conference on Empirical Methods in Natural Language Processing. 2008, 254\u2013263","DOI":"10.3115\/1613715.1613751"},{"key":"2245_CR5","unstructured":"Lu X, Chow T W S. Modeling sequential annotations for sequence labeling with crowds. IEEE Transactions on Cybernetics, 2021: 1\u201311"},{"key":"2245_CR6","doi-asserted-by":"crossref","unstructured":"Lin C H, Mausam, Weld D S. Dynamically switching between synergistic workflows for crowdsourcing. In: Proceedings of the 26th AAAI Conference on Artificial Intelligence. 2012, 87\u201393","DOI":"10.1609\/aaai.v26i1.8121"},{"issue":"11","key":"2245_CR7","doi-asserted-by":"publisher","first-page":"1483","DOI":"10.14778\/2350229.2350263","volume":"5","author":"J Wang","year":"2012","unstructured":"Wang J, Kraska T, Franklin M J, Feng J. CrowdER: crowdsourcing entity resolution. Proceedings of the VLDB Endowment, 2012, 5(11): 1483\u20131494","journal-title":"Proceedings of the VLDB Endowment"},{"issue":"47","key":"2245_CR8","doi-asserted-by":"publisher","first-page":"18949","DOI":"10.1073\/pnas.1115898108","volume":"108","author":"F Khatib","year":"2011","unstructured":"Khatib F, Cooper S, Tyka M D, Xu K, Makedon I, Popovi\u0107 Z, Baker D, Players F. Algorithm discovery by protein folding game players. Proceedings of the National Academy of Sciences of the United States of America, 2011, 108(47): 18949\u201318953","journal-title":"Proceedings of the National Academy of Sciences of the United States of America"},{"key":"2245_CR9","unstructured":"Zaidan O F, Callison-Burch C. Crowdsourcing translation: professional quality from non-professionals. In: Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies. 2011, 1220\u20131229"},{"key":"2245_CR10","doi-asserted-by":"crossref","unstructured":"Murphy M J, Miller C D, Lasecki W S, Bigham J P. Adaptive time windows for real-time crowd captioning. In: Proceedings of CHI\u2019 13 Extended Abstracts on Human Factors in Computing Systems. 2013, 13\u201318","DOI":"10.1145\/2468356.2468360"},{"issue":"1","key":"2245_CR11","first-page":"20","volume":"28","author":"A P Dawid","year":"1979","unstructured":"Dawid A P, Skene A M. Maximum likelihood estimation of observer error-rates using the EM algorithm. Journal of the Royal Statistical Society. Series C (Applied Statistics), 1979, 28(1): 20\u201328","journal-title":"Journal of the Royal Statistical Society. Series C (Applied Statistics)"},{"issue":"3","key":"2245_CR12","doi-asserted-by":"publisher","first-page":"794","DOI":"10.1109\/TKDE.2014.2327026","volume":"27","author":"A Kurve","year":"2015","unstructured":"Kurve A, Miller D J, Kesidis G. Multicategory crowdsourcing accounting for variable task difficulty, worker skill, and worker intention. IEEE Transactions on Knowledge and Data Engineering, 2015, 27(3): 794\u2013809","journal-title":"IEEE Transactions on Knowledge and Data Engineering"},{"key":"2245_CR13","unstructured":"Zhao Z, Wei F, Zhou M, Chen W, Ng W S H. Crowd-selection query processing in crowdsourcing databases: a task-driven approach. In: Proceedings of the 18th International Conference on Extending Database Technology. 2015, 397\u2013408"},{"key":"2245_CR14","doi-asserted-by":"crossref","unstructured":"Lof C, El Maarry K, Balke W T. Skyline queries in crowd-enabled databases. In: Proceedings of the 16th International Conference on Extending Database Technology. 2013, 465\u2013476","DOI":"10.1145\/2452376.2452431"},{"key":"2245_CR15","unstructured":"Chen X, Lin Q, Zhou D. Optimistic knowledge gradient policy for optimal budget allocation in crowdsourcing. In: Proceedings of the 30th International Conference on International Conference on Machine Learning. 2013, III-64\u2013III-72"},{"key":"2245_CR16","doi-asserted-by":"crossref","unstructured":"Fan J, Lu M, Ooi B C, Tan W C, Zhang M. A hybrid machine-crowdsourcing system for matching web tables. In: Proceedings of the 30th IEEE International Conference on Data Engineering. 2014, 976\u2013987","DOI":"10.1109\/ICDE.2014.6816716"},{"issue":"2","key":"2245_CR17","doi-asserted-by":"publisher","first-page":"486","DOI":"10.1109\/TCYB.2016.2642999","volume":"48","author":"M Yousefnezhad","year":"2018","unstructured":"Yousefnezhad M, Huang S J, Zhang D. WoCE: a framework for clustering ensemble by exploiting the wisdom of crowds theory. IEEE Transactions on Cybernetics, 2018, 48(2): 486\u2013499","journal-title":"IEEE Transactions on Cybernetics"},{"issue":"5","key":"2245_CR18","doi-asserted-by":"publisher","first-page":"749","DOI":"10.1109\/JAS.2022.105434","volume":"9","author":"J Zhang","year":"2022","unstructured":"Zhang J. Knowledge learning with crowdsourcing: a brief review and systematic perspective. IEEE\/CAA Journal of Automatica Sinica, 2022, 9(5): 749\u2013762","journal-title":"IEEE\/CAA Journal of Automatica Sinica"},{"key":"2245_CR19","doi-asserted-by":"crossref","unstructured":"Jiang L, Zhang H, Tao F, Li C. Learning from crowds with multiple noisy label distribution propagation. IEEE Transactions on Neural Networks and Learning Systems, 2021: 1\u201311","DOI":"10.1109\/TNNLS.2021.3082496"},{"key":"2245_CR20","doi-asserted-by":"publisher","first-page":"104474","DOI":"10.1016\/j.engappai.2021.104474","volume":"106","author":"F Tao","year":"2021","unstructured":"Tao F, Jiang L, Li C. Differential evolution-based weighted soft majority voting for crowdsourcing. Engineering Applications of Artificial Intelligence, 2021, 106: 104474","journal-title":"Engineering Applications of Artificial Intelligence"},{"issue":"9","key":"2245_CR21","doi-asserted-by":"publisher","first-page":"2246","DOI":"10.1109\/TKDE.2016.2555805","volume":"28","author":"A I Chittilappilly","year":"2016","unstructured":"Chittilappilly A I, Chen L, Amer-Yahia S. A survey of general-purpose crowdsourcing techniques. IEEE Transactions on Knowledge and Data Engineering, 2016, 28(9): 2246\u20132266","journal-title":"IEEE Transactions on Knowledge and Data Engineering"},{"issue":"4","key":"2245_CR22","doi-asserted-by":"publisher","first-page":"543","DOI":"10.1007\/s10462-016-9491-9","volume":"46","author":"J Zhang","year":"2016","unstructured":"Zhang J, Wu X, Sheng V S. Learning from crowdsourced labeled data: a survey. Artificial Intelligence Review, 2016, 46(4): 543\u2013576","journal-title":"Artificial Intelligence Review"},{"key":"2245_CR23","unstructured":"Yan Y, Rosales R, Fung G, Dy J G. Active learning from crowds. In: Proceedings of the 28th International Conference on Machine Learning. 2011, 1161\u20131168"},{"key":"2245_CR24","unstructured":"Mozafari B, Sarkar P, Franklin M J, Jordan M I, Madden S. Active learning for crowd-sourced databases. 2014, arXiv preprint arXiv: 1209.3686"},{"key":"2245_CR25","first-page":"1297","volume":"11","author":"V C Raykar","year":"2010","unstructured":"Raykar V C, Yu S, Zhao L H, Valadez G H, Florin C, Bogoni L, Moy L. Learning from crowds. Journal of Machine Learning Research, 2010, 11: 1297\u20131322","journal-title":"Journal of Machine Learning Research"},{"issue":"5","key":"2245_CR26","doi-asserted-by":"publisher","first-page":"1095","DOI":"10.1109\/TCYB.2014.2344674","volume":"45","author":"J Zhang","year":"2015","unstructured":"Zhang J, Wu X, Shengs V S. Active learning with imbalanced multiple noisy labeling. IEEE Transactions on Cybernetics, 2015, 45(5): 1095\u20131107","journal-title":"IEEE Transactions on Cybernetics"},{"key":"2245_CR27","unstructured":"Bachrach Y, Minka T, Guiver J, Graepel T. How to grade a test without knowing the answers: a Bayesian graphical model for adaptive crowdsourcing and aptitude testing. In: Proceedings of the 29th International Conference on Machine Learning. 2012, 819\u2013826"},{"key":"2245_CR28","unstructured":"Ho C J, Jabbari S, Vaughan J W. Adaptive task assignment for crowdsourced classification. In: Proceedings of the 30th International Conference on Machine Learning. 2013, I-534\u2013I-542"},{"key":"2245_CR29","doi-asserted-by":"crossref","unstructured":"Buchbinder N, Naor J. Online primal-dual algorithms for covering and packing problems. In: Proceedings of 13th Annual European Symposium on Algorithms. 2005, 689\u2013701","DOI":"10.1007\/11561071_61"},{"key":"2245_CR30","doi-asserted-by":"crossref","unstructured":"Long C, Hua G, Kapoor A. Active visual recognition with expertise estimation in crowdsourcing. In: Proceedings of 2013 IEEE International Conference on Computer Vision. 2013, 3000\u20133007","DOI":"10.1109\/ICCV.2013.373"},{"key":"2245_CR31","doi-asserted-by":"crossref","unstructured":"Donmez P, Carbonell J G, Schneider J. Efficiently learning the accuracy of labeling sources for selective sampling. In: Proceedings of the 15th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining. 2009, 259\u2013268","DOI":"10.1145\/1557019.1557053"},{"key":"2245_CR32","unstructured":"Zhao L, Zhang Y, Sukthankar G. An active learning approach for jointly estimating worker performance and annotation reliability with crowdsourced data. 2014, arXiv preprint arXiv: 1401.3836"},{"key":"2245_CR33","doi-asserted-by":"crossref","unstructured":"Lewis D D, Gale W A. A sequential algorithm for training text classifiers. In: Proceedings of the 17th Annual International ACM-SIGIR Conference on Research and Development in Information Retrieval. 1994, 3\u201312","DOI":"10.1007\/978-1-4471-2099-5_1"},{"key":"2245_CR34","volume-title":"Sampling Techniques","author":"W G Cochran","year":"1977","unstructured":"Cochran W G. Sampling Techniques. 3rd ed. Hoboken: John Wiley & Sons, 1977","edition":"3rd ed."},{"key":"2245_CR35","first-page":"355","volume-title":"Readings in Information Retrieval","author":"G Salton","year":"1997","unstructured":"Salton G, Buckley C. Improving retrieval performance by relevance feedback. In: Jones K S, Willett P, eds. Readings in Information Retrieval. San Francisco: Morgan Kaufmann, 1997, 355\u2013364"},{"key":"2245_CR36","doi-asserted-by":"crossref","unstructured":"Dagan I, Engelson S P. Committee-based sampling for training probabilistic classifiers. In: Proceedings of the 12th International Conference on Machine Learning. 1995, 150\u2013157","DOI":"10.1016\/B978-1-55860-377-6.50027-X"},{"issue":"3","key":"2245_CR37","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1145\/1753783.1753784","volume":"6","author":"J Zhu","year":"2010","unstructured":"Zhu J, Wang H, Hovy E, Ma M. Confidence-based stopping criteria for active learning for data annotation. ACM Transactions on Speech and Language Processing, 2010, 6(3): 3","journal-title":"ACM Transactions on Speech and Language Processing"},{"key":"2245_CR38","unstructured":"Dua D, Graff C. UCI machine learning repository. See Archive.ics.uci.eduml website, 2017"},{"key":"2245_CR39","unstructured":"Karger D R, Oh S, Shah D. Iterative learning for reliable crowdsourcing systems. In: Proceedings of the 24th International Conference on Neural Information Processing Systems. 2011, 1953\u20131961"},{"key":"2245_CR40","doi-asserted-by":"crossref","unstructured":"Kajino H, Tsuboi Y, Kashima H. Clustering crowds. In: Proceedings of the 27th AAAI Conference on Artificial Intelligence. 2013, 1120\u20131127","DOI":"10.1609\/aaai.v27i1.8456"},{"key":"2245_CR41","doi-asserted-by":"crossref","unstructured":"Yin L A, Han J H, Zhang W N, Yu Y. Aggregating crowd wisdoms with label-aware autoencoders. In: Proceedings of the 26th International Joint Conference on Artificial Intelligence. 2017, 1325\u20131331","DOI":"10.24963\/ijcai.2017\/184"},{"key":"2245_CR42","unstructured":"Liu Y F, Zhang W N, Yu Y. Aggregating crowd wisdom with side information via a clustering-based label-aware autoencoder. In: Proceedings of the 29th International Joint Conference on Artificial Intelligence. 2021, 214"},{"key":"2245_CR43","first-page":"1","volume-title":"Decision Making and Imperfection","author":"E Simpson","year":"2013","unstructured":"Simpson E, Roberts S, Psorakis I, Smith A. Dynamic Bayesian combination of multiple imperfect classifiers. In: Guy T V, Karny M, Wolpert D, eds. Decision Making and Imperfection. Berlin: Springer, 2013, 1\u201335"}],"container-title":["Frontiers of Computer Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11704-022-2245-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11704-022-2245-8\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11704-022-2245-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,18]],"date-time":"2024-11-18T21:29:55Z","timestamp":1731965395000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11704-022-2245-8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,12,27]]},"references-count":43,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2023,10]]}},"alternative-id":["2245"],"URL":"https:\/\/doi.org\/10.1007\/s11704-022-2245-8","relation":{},"ISSN":["2095-2228","2095-2236"],"issn-type":[{"type":"print","value":"2095-2228"},{"type":"electronic","value":"2095-2236"}],"subject":[],"published":{"date-parts":[[2022,12,27]]},"assertion":[{"value":"26 April 2022","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 September 2022","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"27 December 2022","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"175332"}}