{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,5]],"date-time":"2025-12-05T12:11:40Z","timestamp":1764936700558},"reference-count":49,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2014,2,15]],"date-time":"2014-02-15T00:00:00Z","timestamp":1392422400000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Appl Intell"],"published-print":{"date-parts":[[2014,7]]},"DOI":"10.1007\/s10489-014-0516-2","type":"journal-article","created":{"date-parts":[[2014,2,14]],"date-time":"2014-02-14T01:44:20Z","timestamp":1392342260000},"page":"212-222","source":"Crossref","is-referenced-by-count":13,"title":["Agreement\/disagreement based crowd labeling"],"prefix":"10.1007","volume":"41","author":[{"given":"Hossein","family":"Amirkhani","sequence":"first","affiliation":[]},{"given":"Mohammad","family":"Rahmati","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2014,2,15]]},"reference":[{"key":"516_CR1","first-page":"1","volume":"14","author":"J Howe","year":"2006","unstructured":"Howe J (2006) The rise of crowdsourcing. Wired Mag 14:1\u20134","journal-title":"Wired Mag"},{"key":"516_CR2","unstructured":"Pontin J (2007) Artificial intelligence: with help from the humans. The New York Times"},{"key":"516_CR3","doi-asserted-by":"crossref","first-page":"184","DOI":"10.1007\/s11263-012-0564-1","volume":"101","author":"C Vondrick","year":"2013","unstructured":"Vondrick C, Patterson D, Ramanan D (2013) Efficiently scaling up crowdsourced video annotation. Int J Comput Vis 101:184\u2013204","journal-title":"Int J Comput Vis"},{"key":"516_CR4","doi-asserted-by":"crossref","first-page":"132","DOI":"10.1007\/978-3-642-36415-0_9","volume-title":"Information retrieval meets information visualization","author":"GJ Jones","year":"2013","unstructured":"Jones GJ (2013) An introduction to crowdsourcing for language and multimedia technology research. In: Information retrieval meets information visualization. Springer, Berlin, pp 132\u2013154"},{"key":"516_CR5","first-page":"211","volume-title":"5th biennial conference on innovative data systems research","author":"A Marcus","year":"2011","unstructured":"Marcus A, Wu E, Karger DR et al (2011) Crowdsourced databases: query processing with people. In: 5th biennial conference on innovative data systems research, Asilomar, CA, USA, pp\u00a0211\u2013214"},{"key":"516_CR6","doi-asserted-by":"crossref","first-page":"254","DOI":"10.3115\/1613715.1613751","volume-title":"The conference on empirical methods in natural language processing","author":"R Snow","year":"2008","unstructured":"Snow R, O\u2019Connor B, Jurafsky D, Ng A (2008) Cheap and fast\u2014but is it good? Evaluating non-expert annotations for natural language tasks. In: The conference on empirical methods in natural language processing, pp 254\u2013263"},{"key":"516_CR7","first-page":"1220","volume-title":"Proceedings of the 49th annual meeting of the association for computational linguistics: human language technologies","author":"OF Zaidan","year":"2011","unstructured":"Zaidan OF, Callison-Burch C (2011) Crowdsourcing translation: professional quality from non-professionals. In: Proceedings of the 49th annual meeting of the association for computational linguistics: human language technologies, pp 1220\u20131229"},{"key":"516_CR8","doi-asserted-by":"crossref","first-page":"313","DOI":"10.1145\/1866029.1866078","volume-title":"Proceedings of the 23nd annual ACM symposium on user interface software and technology","author":"MS Bernstein","year":"2010","unstructured":"Bernstein MS, Little G, Miller RC et al (2010) Soylent: a word processor with a crowd inside. In: Proceedings of the 23nd annual ACM symposium on user interface software and technology, pp\u00a0313\u2013322"},{"key":"516_CR9","first-page":"9","volume-title":"ACM SIGIR workshop on crowdsourcing for search evaluation","author":"J Urbano","year":"2010","unstructured":"Urbano J, Morato J, Marrero M, Mart\u00edn D (2010) Crowdsourcing preference judgments for evaluation of music similarity tasks. In: ACM SIGIR workshop on crowdsourcing for search evaluation, pp 9\u201316"},{"key":"516_CR10","first-page":"172","volume-title":"Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with Amazon\u2019s mechanical turk","author":"C Grady","year":"2010","unstructured":"Grady C, Lease M (2010) Crowdsourcing document relevance assessment with mechanical turk. In: Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with Amazon\u2019s mechanical turk, pp 172\u2013179"},{"issue":"321","key":"516_CR11","first-page":"1465","volume":"5895","author":"LV Ahn","year":"2008","unstructured":"Ahn LV, Maurer B, McMillen C et al (2008) Re-CAPTCHA: human-based character recognition via web security measures. Science 5895(321):1465\u20131468","journal-title":"Science"},{"key":"516_CR12","unstructured":"Muhammadi J, Rabiee HR (2013) Crowd computing: a survey. arXiv:1301.2774"},{"key":"516_CR13","doi-asserted-by":"crossref","first-page":"614","DOI":"10.1145\/1401890.1401965","volume-title":"Proceedings of the 14th ACM SIGKDD international conference on knowledge discovery and data mining","author":"VS Sheng","year":"2008","unstructured":"Sheng VS, Provost F, Ipeirotis PG (2008) Get another label? Improving data quality and data mining using multiple, noisy labelers. In: Proceedings of the 14th ACM SIGKDD international conference on knowledge discovery and data mining, pp 614\u2013622"},{"key":"516_CR14","first-page":"233","volume-title":"Proceedings of the 26th international conference on machine learning","author":"O Dekel","year":"2009","unstructured":"Dekel O, Shamir O (2009) Good learners for evil teachers. In: Proceedings of the 26th international conference on machine learning, Montreal, Canada, pp 233\u2013240"},{"key":"516_CR15","first-page":"1297","volume":"11","author":"VC Raykar","year":"2010","unstructured":"Raykar VC, Yu S, Zhao LH et al (2010) Learning from crowds. J\u00a0Mach Learn Res 11:1297\u20131322","journal-title":"J\u00a0Mach Learn Res"},{"key":"516_CR16","first-page":"2035","volume-title":"Advances in neural information processing systems","author":"J Whitehill","year":"2009","unstructured":"Whitehill J, Ruvolo P, Wu T et al (2009) Whose vote should count more: optimal integration of labels from labelers of unknown expertise. In: Advances in neural information processing systems, pp\u00a02035\u20132043"},{"key":"516_CR17","first-page":"491","volume":"13","author":"VC Raykar","year":"2012","unstructured":"Raykar VC, Yu S (2012) Eliminating spammers and ranking annotators for crowdsourced labeling tasks. J Mach Learn Res 13:491\u2013518","journal-title":"J Mach Learn Res"},{"key":"516_CR18","doi-asserted-by":"crossref","first-page":"64","DOI":"10.1145\/1837885.1837906","volume-title":"The ACM SIGKDD workshop on human computation","author":"PG Ipeirotis","year":"2010","unstructured":"Ipeirotis PG, Provost F, Wang J (2010) Quality management on amazon mechanical turk. In: The ACM SIGKDD workshop on human computation. ACM, New York, pp 64\u201367"},{"key":"516_CR19","doi-asserted-by":"crossref","first-page":"20","DOI":"10.2307\/2346806","volume":"28","author":"AP Dawid","year":"1979","unstructured":"Dawid AP, Skene AM (1979) Maximum likelihood estimation of observer error-rates using the EM algorithm. Appl Stat 28:20\u201328","journal-title":"Appl Stat"},{"key":"516_CR20","volume-title":"Neural information processing systems (NIPS)","author":"DR Karger","year":"2011","unstructured":"Karger DR, Oh S, Shah D (2011) Iterative learning for reliable crowdsourcing systems. In: Neural information processing systems (NIPS)"},{"key":"516_CR21","volume-title":"Proceedings of the NIPS 2nd workshop on computational social science and the wisdom of crowds","author":"FK Khattak","year":"2011","unstructured":"Khattak FK, Salleb-Aouissi A (2011) Quality control of crowd labeling through expert evaluation. In: Proceedings of the NIPS 2nd workshop on computational social science and the wisdom of crowds"},{"key":"516_CR22","volume-title":"Neural information processing systems (NIPS)","author":"FL Wauthier","year":"2011","unstructured":"Wauthier FL, Jordan MI (2011) Bayesian bias mitigation for crowdsourcing. In: Neural information processing systems (NIPS)"},{"key":"516_CR23","first-page":"1085","volume-title":"Advances in neural information processing systems","author":"P Smyth","year":"1995","unstructured":"Smyth P, Fayyad U, Burl M et al (1995) Inferring ground truth from subjective labelling of venus images. In: Advances in neural information processing systems, pp 1085\u20131092"},{"key":"516_CR24","doi-asserted-by":"crossref","first-page":"246","DOI":"10.3115\/1034678.1034721","volume-title":"Proceedings of the 37th annual meeting of the association for computational linguistics on computational linguistics","author":"JM Wiebe","year":"1999","unstructured":"Wiebe JM, Bruce RF, O\u2019Hara TP (1999) Development and use of a gold-standard data set for subjectivity classifications. In: Proceedings of the 37th annual meeting of the association for computational linguistics on computational linguistics, pp 246\u2013253"},{"key":"516_CR25","doi-asserted-by":"crossref","first-page":"447","DOI":"10.1007\/978-3-642-02767-3_50","volume-title":"Internationalization, design and global development","author":"N Eagle","year":"2009","unstructured":"Eagle N (2009) Txteagle: mobile crowdsourcing. In: Internationalization, design and global development. Springer, Berlin, pp 447\u2013456"},{"key":"516_CR26","first-page":"932","volume-title":"International conference on artificial intelligence and statistics","author":"Y Yan","year":"2010","unstructured":"Yan Y, Rosales R, Fung G et al (2010) Modeling annotator expertise: learning when everybody knows a bit of something. In: International conference on artificial intelligence and statistics, pp\u00a0932\u2013939"},{"key":"516_CR27","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1111\/j.2517-6161.1977.tb01600.x","volume":"39","author":"AP Dempster","year":"1977","unstructured":"Dempster AP, Laird NM, Rubin DB (1977) Maximum likelihood from incomplete data via the EM algorithm. J R Stat Soc, Ser B 39:1\u201338","journal-title":"J R Stat Soc, Ser B"},{"key":"516_CR28","first-page":"73","volume-title":"Proceedings of the 26th AAAI conference on artificial intelligence","author":"H Kajino","year":"2012","unstructured":"Kajino H, Tsuboi Y, Kashima H (2012) A convex formulation for learning from crowds. In: Proceedings of the 26th AAAI conference on artificial intelligence, pp 73\u201379"},{"key":"516_CR29","unstructured":"Karger DR, Oh S, Shah D (2012) Budget-optimal task allocation for reliable crowdsourcing systems. arXiv:1110.3564v3"},{"key":"516_CR30","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1007\/3-540-45014-9_1","volume-title":"Multiple classifier systems","author":"TG Dietterich","year":"2000","unstructured":"Dietterich TG (2000) Ensemble methods in machine learning. In: Multiple classifier systems. Springer, Berlin, pp 1\u201315"},{"key":"516_CR31","doi-asserted-by":"crossref","first-page":"139","DOI":"10.1023\/A:1007607513941","volume":"40","author":"TG Dietterich","year":"2000","unstructured":"Dietterich TG (2000) An experimental comparison of three methods for constructing ensembles of decision trees: bagging, boosting, and randomization. Mach Learn 40:139\u2013157","journal-title":"Mach Learn"},{"key":"516_CR32","first-page":"725","volume":"5","author":"G Valentini","year":"2004","unstructured":"Valentini G, Dietterich TG (2004) Bias-variance analysis of support vector machines for the development of SVM-based ensemble methods. J Mach Learn Res 5:725\u2013775","journal-title":"J Mach Learn Res"},{"key":"516_CR33","doi-asserted-by":"crossref","first-page":"243","DOI":"10.1007\/s10489-006-0020-4","volume":"26","author":"SB Cho","year":"2007","unstructured":"Cho SB, Won H-H (2007) Cancer classification using ensemble of neural networks with multiple significant gene subsets. Appl Intell 26:243\u2013250","journal-title":"Appl Intell"},{"key":"516_CR34","doi-asserted-by":"crossref","first-page":"536","DOI":"10.1007\/978-3-642-03348-3_54","volume-title":"Advanced data mining and applications","author":"Z Lin","year":"2009","unstructured":"Lin Z, Hao Z, Yang X, Liu X (2009) Several SVM ensemble methods integrated with under-sampling for imbalanced data learning. In: Advanced data mining and applications. Springer, Berlin, pp\u00a0536\u2013544"},{"key":"516_CR35","doi-asserted-by":"crossref","first-page":"347","DOI":"10.1007\/s10489-011-0283-2","volume":"34","author":"J Maudes","year":"2011","unstructured":"Maudes J, Rodr\u00edguez JJ, Garc\u00eda-Osorio C, Pardo C (2011) Random projections for linear SVM ensembles. Appl Intell 34:347\u2013359","journal-title":"Appl Intell"},{"key":"516_CR36","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1007\/s10489-009-0199-2","volume":"35","author":"AM Canuto","year":"2011","unstructured":"Canuto AM, Santos AM, Vargas RR (2011) Ensembles of ARTMAP-based neural networks: an experimental study. Appl Intell 35:1\u201317","journal-title":"Appl Intell"},{"key":"516_CR37","doi-asserted-by":"crossref","first-page":"320","DOI":"10.1007\/s10489-010-0263-y","volume":"36","author":"K-C Khor","year":"2012","unstructured":"Khor K-C, Ting C-Y, Phon-Amnuaisuk S (2012) A cascaded classifier approach for improving detection rates on rare attack categories in network intrusion detection. Appl Intell 36:320\u2013329","journal-title":"Appl Intell"},{"key":"516_CR38","doi-asserted-by":"crossref","first-page":"488","DOI":"10.1007\/s10489-012-0342-3","volume":"37","author":"H Lee","year":"2012","unstructured":"Lee H, Kim E, Pedrycz W (2012) A new selective neural network ensemble with negative correlation. Appl Intell 37:488\u2013498","journal-title":"Appl Intell"},{"issue":"3","key":"516_CR39","doi-asserted-by":"crossref","first-page":"465","DOI":"10.1007\/s10489-013-0425-9","volume":"39","author":"C-W Wang","year":"2013","unstructured":"Wang C-W, You W-H (2013) Boosting-SVM: effective learning with reduced data dimension. Appl Intell 39(3):465\u2013474","journal-title":"Appl Intell"},{"issue":"2","key":"516_CR40","doi-asserted-by":"crossref","first-page":"244","DOI":"10.1007\/s10489-013-0457-1","volume":"40","author":"S Park","year":"2014","unstructured":"Park S, Lee SR (2014) Red tides prediction system using fuzzy reasoning and the ensemble method. Appl Intell 40(2):244\u2013255","journal-title":"Appl Intell"},{"issue":"4","key":"516_CR41","doi-asserted-by":"crossref","first-page":"566","DOI":"10.1007\/s10489-012-0388-2","volume":"38","author":"A Bella","year":"2013","unstructured":"Bella A, Ferri C, Hern\u00e1ndez-Orallo J, Ram\u00edrez-Quintana MJ (2013) On the effect of calibration in classifier combination. Appl Intell 38(4):566\u2013585","journal-title":"Appl Intell"},{"issue":"2","key":"516_CR42","doi-asserted-by":"crossref","first-page":"291","DOI":"10.1007\/s10489-013-0464-2","volume":"40","author":"CO Sakar","year":"2014","unstructured":"Sakar CO, Kursun O, Gurgen F (2014) Ensemble canonical correlation analysis. Appl Intell 40(2):291\u2013304","journal-title":"Appl Intell"},{"key":"516_CR43","doi-asserted-by":"crossref","first-page":"88","DOI":"10.1007\/s10489-012-0359-7","volume":"38","author":"M Fahim","year":"2013","unstructured":"Fahim M, Fatima I, Lee S, Lee Y-K (2013) EEM: evolutionary ensembles model for activity recognition in smart homes. Appl Intell 38:88\u201398","journal-title":"Appl Intell"},{"key":"516_CR44","doi-asserted-by":"crossref","first-page":"205","DOI":"10.1080\/088395100117124","volume":"14","author":"D Gamberger","year":"2000","unstructured":"Gamberger D, Lavrac N, Dzeroski S (2000) Noise detection and elimination in data preprocessing: experiments in medical domains. Appl Artif Intell 14:205\u2013223","journal-title":"Appl Artif Intell"},{"key":"516_CR45","doi-asserted-by":"crossref","first-page":"5","DOI":"10.1007\/3-540-48050-1_3","volume-title":"Foundations of intelligent systems","author":"S Lallich","year":"2002","unstructured":"Lallich S, Muhlenbach F, Zighed DA (2002) Improving classification by removing or relabeling mislabeled instances. In: Foundations of intelligent systems. Springer, Berlin, pp 5\u201315"},{"key":"516_CR46","doi-asserted-by":"crossref","first-page":"317","DOI":"10.1007\/3-540-44938-8_32","volume-title":"Multiple classifier systems","author":"S Verbaeten","year":"2003","unstructured":"Verbaeten S, Van Assche A (2003) Ensemble methods for noise elimination in classification problems. In: Multiple classifier systems. Springer, Berlin, pp 317\u2013325"},{"key":"516_CR47","doi-asserted-by":"crossref","first-page":"345","DOI":"10.1007\/s10489-010-0225-4","volume":"35","author":"D Guan","year":"2011","unstructured":"Guan D, Yuan W, Lee Y-K, Lee S (2011) Identifying mislabeled training data with the aid of unlabeled data. Appl Intell 35:345\u2013358","journal-title":"Appl Intell"},{"key":"516_CR48","doi-asserted-by":"crossref","first-page":"170","DOI":"10.1109\/PRNI.2013.51","volume-title":"International workshop on pattern recognition in neuroimaging (PRNI)","author":"J Young","year":"2013","unstructured":"Young J, Ashburner J, Ourselin S (2013) Wrapper methods to correct mislabelled training data. In: International workshop on pattern recognition in neuroimaging (PRNI), pp 170\u2013173"},{"key":"516_CR49","volume-title":"Statistical inference","author":"L Wasserman","year":"2003","unstructured":"Wasserman L (2003) All of statistics: a concise course. In: Statistical inference. Springer, Berlin"}],"container-title":["Applied Intelligence"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-014-0516-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10489-014-0516-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-014-0516-2","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,5,24]],"date-time":"2024-05-24T07:09:49Z","timestamp":1716534589000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10489-014-0516-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014,2,15]]},"references-count":49,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2014,7]]}},"alternative-id":["516"],"URL":"https:\/\/doi.org\/10.1007\/s10489-014-0516-2","relation":{},"ISSN":["0924-669X","1573-7497"],"issn-type":[{"value":"0924-669X","type":"print"},{"value":"1573-7497","type":"electronic"}],"subject":[],"published":{"date-parts":[[2014,2,15]]}}}