{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,20]],"date-time":"2026-01-20T04:07:06Z","timestamp":1768882026426,"version":"3.49.0"},"reference-count":32,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2015,7,11]],"date-time":"2015-07-11T00:00:00Z","timestamp":1436572800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J Intell Inf Syst"],"published-print":{"date-parts":[[2016,12]]},"DOI":"10.1007\/s10844-015-0371-6","type":"journal-article","created":{"date-parts":[[2015,7,10]],"date-time":"2015-07-10T02:24:44Z","timestamp":1436495084000},"page":"403-425","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["A formalized framework for incorporating expert labels in crowdsourcing environment"],"prefix":"10.1007","volume":"47","author":[{"given":"Qingyang","family":"Hu","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Qinming","family":"He","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hao","family":"Huang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kevin","family":"Chiew","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhenguang","family":"Liu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2015,7,11]]},"reference":[{"key":"371_CR1","unstructured":"Ambati, V., Vogel, S., & Carbonell, J. (2010). Active learning and crowd-sourcing for machine translation. In Proceedings of the 7th International Conference on Language Resources and Evaluation (LREC), (Vol. 7 pp. 2169\u20132174): Citeseer."},{"key":"371_CR2","unstructured":"Bishop, C.M., & et al. (2006). Pattern recognition and machine learning Vol. 1. New York: Springer."},{"key":"371_CR3","unstructured":"Brew, A., Greene, D., & Cunningham, P. (2010). Using crowdsourcing and active learning to track sentiment in online media. In Proceedings of the 19th European Conference on Artificial Intelligence (ECAI) (pp. 145\u2013150)."},{"key":"371_CR4","doi-asserted-by":"crossref","unstructured":"Dagan, I., & Engelson, S.P. (1995). Committee-based sampling for training probabilistic classifiers. In Proceedings of the International Conference on Machine Learning (ICML\u201995), (Vol. 95 pp. 150\u2013157): Citeseer.","DOI":"10.1016\/B978-1-55860-377-6.50027-X"},{"issue":"1","key":"371_CR5","doi-asserted-by":"crossref","first-page":"20","DOI":"10.2307\/2346806","volume":"28","author":"AP Dawid","year":"1979","unstructured":"Dawid, A.P., & Skene, A.M. (1979). Maximum likelihood estimation of observer error-rates using the EM algorithm. Applied Statistics, 28(1), 20\u201328.","journal-title":"Applied Statistics"},{"key":"371_CR6","doi-asserted-by":"crossref","unstructured":"Donmez, P., Carbonell, J.G., & Schneider, J. (2009). Efficiently learning the accuracy of labeling sources for selective sampling. In Proceedings of the 15th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (pp. 259\u2013268).","DOI":"10.1145\/1557019.1557053"},{"key":"371_CR7","unstructured":"Finin, T., Murnane, W., Karandikar, A., Keller, N., Martineau, J., & Dredze, M. (2010). Annotating named entities in twitter data with crowdsourcing. In Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk (pp. 80\u201388)."},{"key":"371_CR8","unstructured":"Frank, A., & Asuncion, A. (2010). UCI machine learning repository."},{"key":"371_CR9","doi-asserted-by":"crossref","unstructured":"Hu, Q., He, Q., Huang, H., Chiew, K., & Liu, Z. (2014). Learning from crowds under experts\u2019 supervision. In Proceedings of the 18th Pacific-Asia Conference on Knowledge Discovery and Data Mining (pp. 200\u2013211).","DOI":"10.1007\/978-3-319-06608-0_17"},{"key":"371_CR10","doi-asserted-by":"crossref","unstructured":"Ipeirotis, P.G., Provost, F., & Wang, J. (2010). Quality management on amazon mechanical turk. In Proceedings of the ACM SIGKDD Workshop on Human Computation (pp. 64\u201367).","DOI":"10.1145\/1837885.1837906"},{"key":"371_CR11","unstructured":"Jaakkola, T., & Jordan, M. (1997). A variational approach to Bayesian logistic regression models and their extensions. In Proceedings of the 6th International Workshop on Artificial Intelligence and Statistics."},{"key":"371_CR12","doi-asserted-by":"crossref","unstructured":"Kajino, H., Tsuboi, Y., & Kashima, H. (2012). A convex formulation for learning from crowds. In Proceedings of the 26th AAAI Conference on Artificial Intelligence (pp. 73\u201379).","DOI":"10.1527\/tjsai.27.133"},{"key":"371_CR13","unstructured":"Kajino, H., Tsuboi, Y., Sato, I., & Kashima, H. (2012). Learning from crowds and experts. In Proceedings of the 4th Human Computation Workshop (HCOMP\u201912) (pp. 107\u2013113)."},{"key":"371_CR14","unstructured":"Karger, D.R., Oh, S., & Shah, D. (2011). Iterative learning for reliable crowdsourcing systems. In Proceedings of Advances in Neural Information Processing Systems (NIPS\u201911) (pp. 1953\u20131961)."},{"issue":"3","key":"371_CR15","doi-asserted-by":"crossref","first-page":"964","DOI":"10.1109\/TNN.2007.897478","volume":"18","author":"LI Kuncheva","year":"2007","unstructured":"Kuncheva, L.I. (2007). Combining pattern classifiers: Methods and algorithms (kuncheva, li; 2004)[book review]. IEEE Transactions on Neural Networks, 18(3), 964\u2013964.","journal-title":"IEEE Transactions on Neural Networks"},{"issue":"2","key":"371_CR16","doi-asserted-by":"crossref","first-page":"299","DOI":"10.1016\/S0031-3203(99)00223-X","volume":"34","author":"LI Kuncheva","year":"2001","unstructured":"Kuncheva, L.I., Bezdek, J.C., & Duin, R.P.W. (2001). Decision templates for multiple classifier fusion: an experimental comparison. Pattern Recognition, 34(2), 299\u2013314.","journal-title":"Pattern Recognition"},{"key":"371_CR17","doi-asserted-by":"crossref","unstructured":"Lewis, D.D., & Gale, W.A. (1994). A sequential algorithm for training text classifiers. In Proceedings of the 17th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (pp. 3\u201312).","DOI":"10.1007\/978-1-4471-2099-5_1"},{"key":"371_CR18","unstructured":"Liu, Q., Peng, J., & Ihler, A. (2012). Variational inference for crowdsourcing. In Proceedings of Advances in Neural Information Processing Systems (NIPS\u201912) (pp. 701\u2013709)."},{"key":"371_CR19","unstructured":"McCallumzy, A.K., & Nigamy, K. (1998). Employing em and pool-based active learning for text classification. In Proceedings of the International Conference on Machine Learning (ICML\u201995): Citeseer."},{"issue":"1","key":"371_CR20","doi-asserted-by":"crossref","first-page":"33","DOI":"10.1023\/A:1007559205422","volume":"36","author":"CJ Merz","year":"1999","unstructured":"Merz, C.J. (1999). Using correspondence analysis to combine classifiers. Machine Learning, 36(1), 33\u201358.","journal-title":"Machine Learning"},{"key":"371_CR21","unstructured":"Raykar, V.C., & Yu, S. (2011). Ranking annotators for crowdsourced labeling tasks. In Proceedings of Advances in Neural Information Processing Systems (NIPS\u201911) (pp. 1809\u20131817)."},{"issue":"4","key":"371_CR22","first-page":"1297","volume":"11","author":"VC Raykar","year":"2010","unstructured":"Raykar, V.C., Yu, S., Zhao, L.H., Valadez, G.H., Florin, C., Bogoni, L., & Moy, L. (2010). Learning from crowds. The Journal of Machine Learning Research, 11(4), 1297\u20131322.","journal-title":"The Journal of Machine Learning Research"},{"key":"371_CR23","doi-asserted-by":"crossref","unstructured":"Seung, H.S., Opper, M., & Sompolinsky, H. (1992). Query by committee. In Proceedings of the 5th Annual Workshop on Computational Learning Theory (pp. 287\u2013294): ACM.","DOI":"10.1145\/130385.130417"},{"key":"371_CR24","doi-asserted-by":"crossref","unstructured":"Sheng, V.S., Provost, F., & Ipeirotis, P.G. (2008). Get another label? Improving data quality and data mining using multiple, noisy labelers. In Proceeding of the 14th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (pp. 614\u2013622).","DOI":"10.1145\/1401890.1401965"},{"key":"371_CR25","unstructured":"Snow, R., O\u2019Connor, B., Jurafsky, D., & Ng, A. (2008). Cheap and fast\u2014but is it good? evaluating non-expert annotations for natural language tasks. In Proceedings of the Conference on Empirical Methods in Natural Language Processing (pp. 254\u2013263): Association for Computational Linguistics."},{"key":"371_CR26","unstructured":"Tang, W., & Lease, M. (2011). Semi-supervised consensus labeling for crowdsourcing. In Proceedings of the 2nd ACM SIGIR Workshop on Crowdsourcing for Information Retrieval (CIR\u201911)."},{"key":"371_CR27","doi-asserted-by":"crossref","unstructured":"Tian, Y., & Zhu, J. (2012). Learning from crowds in the presence of schools of thought. In Proceedings of the 18th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (pp. 226\u2013234).","DOI":"10.1145\/2339530.2339571"},{"key":"371_CR28","unstructured":"Wauthier, F.L., & Jordan, M.I. (2011). Bayesian bias mitigation for crowdsourcing. In Proceedings of Advances in Neural Information Processing Systems (NIPS\u201911) (pp. 1800\u20131808)."},{"key":"371_CR29","unstructured":"Welinder, P., Branson, S., Belongie, S., & Perona, P. (2010). The multidimensional wisdom of crowds. In Proceedings of Advances in Neural Information Processing Systems (NIPS\u201910) (pp. 2424\u20132432)."},{"issue":"2","key":"371_CR30","doi-asserted-by":"crossref","first-page":"241","DOI":"10.1016\/S0893-6080(05)80023-1","volume":"5","author":"DH Wolpert","year":"1992","unstructured":"Wolpert, D.H. (1992). Stacked generalization. Neural Networks, 5(2), 241\u2013259.","journal-title":"Neural Networks"},{"key":"371_CR31","unstructured":"Yan, Y., & et al. (2010). Modeling annotator expertise: Learning when everybody knows a bit of something. In Proceedings of 13th International Conference on Artificial Intelligence and Statistics (AISTATS\u201910), (Vol. 9 pp. 932\u2013939)."},{"key":"371_CR32","unstructured":"Yan, Y., Rosales, R., Fung, G., & Dy, J. (2011). Active learning from crowds. In Proceedings of the International Conference on Machine Learning (ICML\u201911) (pp. 1161\u20131168)."}],"container-title":["Journal of Intelligent Information Systems"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10844-015-0371-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10844-015-0371-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10844-015-0371-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10844-015-0371-6","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,5,30]],"date-time":"2019-05-30T22:10:54Z","timestamp":1559254254000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10844-015-0371-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,7,11]]},"references-count":32,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2016,12]]}},"alternative-id":["371"],"URL":"https:\/\/doi.org\/10.1007\/s10844-015-0371-6","relation":{},"ISSN":["0925-9902","1573-7675"],"issn-type":[{"value":"0925-9902","type":"print"},{"value":"1573-7675","type":"electronic"}],"subject":[],"published":{"date-parts":[[2015,7,11]]}}}