{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,31]],"date-time":"2026-01-31T08:38:26Z","timestamp":1769848706990,"version":"3.49.0"},"reference-count":54,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2013,3,16]],"date-time":"2013-03-16T00:00:00Z","timestamp":1363392000000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Data Min Knowl Disc"],"published-print":{"date-parts":[[2014,3]]},"DOI":"10.1007\/s10618-013-0306-1","type":"journal-article","created":{"date-parts":[[2013,3,15]],"date-time":"2013-03-15T23:25:23Z","timestamp":1363389923000},"page":"402-441","source":"Crossref","is-referenced-by-count":118,"title":["Repeated labeling using multiple noisy labelers"],"prefix":"10.1007","volume":"28","author":[{"given":"Panagiotis G.","family":"Ipeirotis","sequence":"first","affiliation":[]},{"given":"Foster","family":"Provost","sequence":"additional","affiliation":[]},{"given":"Victor S.","family":"Sheng","sequence":"additional","affiliation":[]},{"given":"Jing","family":"Wang","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2013,3,16]]},"reference":[{"key":"306_CR1","first-page":"255","volume":"5","author":"Y Baram","year":"2004","unstructured":"Baram Y, El-Yaniv R, Luz K (2004) Online choice of active learning algorithms. J Mach Learn Res 5:255\u2013291","journal-title":"J Mach Learn Res"},{"key":"306_CR2","unstructured":"Blake CL, Merz CJ (1998) UCI repository of machine learning databases. http:\/\/www.ics.uci.edu\/~mlearn\/MLRepository.html . Accessed 11 Mar 2013"},{"issue":"9","key":"306_CR3","doi-asserted-by":"crossref","first-page":"1757","DOI":"10.1016\/j.patcog.2004.03.009","volume":"37","author":"MR Boutell","year":"2004","unstructured":"Boutell MR, Luo J, Shen X, Brown CM (2004) Learning multi-label scene classification. Pattern Recognit 37(9):1757\u20131771","journal-title":"Pattern Recognit"},{"issue":"1","key":"306_CR4","doi-asserted-by":"crossref","first-page":"5","DOI":"10.1023\/A:1010933404324","volume":"45","author":"L Breiman","year":"2001","unstructured":"Breiman L (2001) Random forests. Mach Learn 45(1):5\u201332","journal-title":"Mach Learn"},{"key":"306_CR5","doi-asserted-by":"crossref","first-page":"131","DOI":"10.1613\/jair.606","volume":"11","author":"CE Brodley","year":"1999","unstructured":"Brodley CE, Friedl MA (1999) Identifying mislabeled training data. J Artif Intell Res 11:131\u2013167","journal-title":"J Artif Intell Res"},{"key":"306_CR6","unstructured":"Carpenter B (2008) Multilevel bayesian model of categorical data annotation. http:\/\/lingpipe-blog.com\/lingpipe-white-papers\/ . Accessed 11 Mar 2013"},{"issue":"7","key":"306_CR7","doi-asserted-by":"crossref","first-page":"767","DOI":"10.1287\/mnsc.36.7.767","volume":"36","author":"RT Clemen","year":"1990","unstructured":"Clemen RT, Winkler RL (1990) Unanimity and compromise among probability forecasters. Manag Sci 36(7):767\u2013779","journal-title":"Manag Sci"},{"issue":"2","key":"306_CR8","first-page":"201","volume":"15","author":"DA Cohn","year":"1994","unstructured":"Cohn DA, Atlas LE, Ladner RE (1994) Improving generalization with active learning. Mach Learn 15(2):201\u2013221","journal-title":"Mach Learn"},{"issue":"1","key":"306_CR9","doi-asserted-by":"crossref","first-page":"20","DOI":"10.2307\/2346806","volume":"28","author":"AP Dawid","year":"1979","unstructured":"Dawid AP, Skene AM (1979) Maximum likelihood estimation of observer error-rates using the EM algorithm. Appl Stat 28(1):20\u201328","journal-title":"Appl Stat"},{"key":"306_CR10","doi-asserted-by":"crossref","unstructured":"Domingos P (1999) MetaCost: a general method for making classifiers cost-sensitive. In: Proceedings of the fifth ACM SIGKDD international conference on knowledge discovery and data mining (KDD-99). pp 155\u2013164","DOI":"10.1145\/312129.312220"},{"key":"306_CR11","doi-asserted-by":"crossref","unstructured":"Donmez P, Carbonell JG (2008) Proactive learning: cost-sensitive active learning with multiple imperfect oracles. In: Proceedings of the 17th ACM conference on information and knowledge management (CIKM 2008). pp 619\u2013628","DOI":"10.1145\/1458082.1458165"},{"key":"306_CR12","doi-asserted-by":"crossref","unstructured":"Donmez P, Carbonell JG, Schneider J (2009) Efficiently learning the accuracy of labeling sources for selective sampling. In: Proceedings of the 15th ACM SIGKDD international conference on knowledge discovery and data mining (KDD 2009). pp 259\u2013268","DOI":"10.1145\/1557019.1557053"},{"key":"306_CR13","doi-asserted-by":"crossref","unstructured":"Donmez P, Carbonell JG, Schneider J (2010) A probabilistic framework to learn from multiple annotators with time-varying accuracy. In: Proceedings of the SIAM international conference on data mining (SDM 2010). pp 826\u2013837","DOI":"10.1137\/1.9781611972801.72"},{"key":"306_CR14","unstructured":"Elkan C (2001) The foundations of cost-sensitive learning. In: Proceedings of the seventeenth international joint conference on, artificial intelligence (IJCAI-01). pp 973\u2013978"},{"key":"306_CR15","doi-asserted-by":"crossref","DOI":"10.1201\/9780429258480","volume-title":"Bayesian data analysis","author":"A Gelman","year":"2003","unstructured":"Gelman A, Carlin JB, Stern HS, Rubin DB (2003) Bayesian data analysis, 2nd edn. Chapman and Hall\/CRC, Boca Raton","edition":"2"},{"key":"306_CR16","doi-asserted-by":"crossref","unstructured":"Ipeirotis PG, Provost F, Wang J (2010) Quality management on amazon mechanical turk. In: Proceedings of the ACM SIGKDD workshop on human computation (HCOMP 2010). pp 64\u201367","DOI":"10.1145\/1837885.1837906"},{"key":"306_CR17","unstructured":"Jin R, Ghahramani Z (2002) Learning with multiple labels. In: Advances in neural information processing systems 15 (NIPS 2002). pp 897\u2013904"},{"key":"306_CR18","doi-asserted-by":"crossref","unstructured":"Kapoor A, Greiner R (2005) Learning and classifying under hard budgets. In: ECML 2005, 16th European conference on machine learning. pp 170\u2013181","DOI":"10.1007\/11564096_20"},{"key":"306_CR19","unstructured":"Lizotte DJ, Madani O, Greiner R (2003) Budgeted learning of naive-bayes classifiers. In: 19th conference on uncertainty in artificial intelligence (UAI 2003). pp 378\u2013385"},{"issue":"1","key":"306_CR20","doi-asserted-by":"crossref","first-page":"79","DOI":"10.1016\/0031-3203(92)90008-7","volume":"25","author":"G Lugosi","year":"1992","unstructured":"Lugosi G (1992) Learning with an unreliable teacher. Pattern Recognit 25(1):79\u201387","journal-title":"Pattern Recognit"},{"key":"306_CR21","unstructured":"Margineantu DD (2005) Active cost-sensitive learning. In: Proceedings of the nineteenth international joint conference on, artificial intelligence (IJCAI-05). pp 1622\u20131613"},{"key":"306_CR22","doi-asserted-by":"crossref","unstructured":"Mason W, Watts DJ (2009) Financial incentives and the performance of crowds. In: Proceedings of the human computation workshop (HCOMP 2009). pp 77\u201385","DOI":"10.1145\/1600150.1600175"},{"key":"306_CR23","unstructured":"McCallum A (1999) Multi-label text classification with a mixture model trained by EM. In: AAAI\u201999 workshop on text learning"},{"key":"306_CR24","doi-asserted-by":"crossref","unstructured":"Melville P, Saar-Tsechansky M, Provost FJ, Mooney RJ (2004) Active feature-value acquisition for classifier induction. In: Proceedings of the 4th IEEE international conference on data mining (ICDM 2004). pp 483\u2013486","DOI":"10.1109\/ICDM.2004.10075"},{"key":"306_CR25","doi-asserted-by":"crossref","unstructured":"Melville P, Provost FJ, Mooney RJ (2005) An expected utility approach to active feature-value acquisition. In: Proceedings of the 5th IEEE international conference on data mining (ICDM 2005). pp 745\u2013748","DOI":"10.1109\/ICDM.2005.23"},{"key":"306_CR26","unstructured":"Morrison CT, Cohen PR (2005) Noisy information value in utility-based decision making. In: Proceedings of the 1st international workshop on utility-based data mining (UBDM\u201905). pp 34\u201338"},{"key":"306_CR27","doi-asserted-by":"crossref","unstructured":"Provost F (2005) Toward economic machine learning and utility-based data mining. In: Proceedings of the 1st international workshop on utility-based data mining (UBDM\u201905). p 1","DOI":"10.1145\/1089827.1089841"},{"key":"306_CR28","unstructured":"Provost F, Danyluk AP (1995) Learning from bad data. In: Proceedings of the ML-95 workshop on applying machine learning, in practice. pp 27\u201333"},{"issue":"1","key":"306_CR29","first-page":"81","volume":"1","author":"JR Quinlan","year":"1986","unstructured":"Quinlan JR (1986) Induction of decision trees. Mach Learn 1(1):81\u2013106","journal-title":"Mach Learn"},{"key":"306_CR30","volume-title":"C4.5: programs for machine learning","author":"JR Quinlan","year":"1992","unstructured":"Quinlan JR (1992) C4.5: programs for machine learning. Morgan Kaufmann Publishers Inc, San Mateo"},{"key":"306_CR31","doi-asserted-by":"crossref","unstructured":"Raykar VC, Yu S, Zhao LH, Jerebko A, Florin C, Valadez GH, Bogoni L, Moy L (2009) Supervised learning from multiple experts: whom to trust when everyone lies a bit. In: Proceedings of the 26th annual international conference on machine learning (ICML 2009). pp. 889\u2013896","DOI":"10.1145\/1553374.1553488"},{"issue":"7","key":"306_CR32","first-page":"1297","volume":"11","author":"VC Raykar","year":"2010","unstructured":"Raykar VC, Yu S, Zhao LH, Valadez GH, Florin C, Bogoni L, Moy L (2010) Learning from crowds. J Mach Learn Res 11(7):1297\u20131322","journal-title":"J Mach Learn Res"},{"key":"306_CR33","doi-asserted-by":"crossref","unstructured":"Rebbapragada U, Brodley CE (2007) Class noise mitigation through instance weighting. In: 18th European conference on machine learning (ECML\u201907). pp. 708\u2013715","DOI":"10.1007\/978-3-540-74958-5_71"},{"issue":"2","key":"306_CR34","doi-asserted-by":"crossref","first-page":"153","DOI":"10.1023\/B:MACH.0000011806.12374.c3","volume":"54","author":"M Saar-Tsechansky","year":"2004","unstructured":"Saar-Tsechansky M, Provost F (2004) Active sampling for class probability estimation and ranking. Mach Learn 54(2):153\u2013178","journal-title":"Mach Learn"},{"key":"306_CR35","doi-asserted-by":"crossref","unstructured":"Saar-Tsechansky M, Melville P, Provost F (2009) Active feature-value acquisition. Manag Sci 55(4):664\u2013684","DOI":"10.1287\/mnsc.1080.0952"},{"key":"306_CR36","doi-asserted-by":"crossref","unstructured":"Sheng VS, Provost F, Ipeirotis P (2008) Get another label? Improving data quality and data mining using multiple, noisy labelers. In: Proceedings of the fourteenth ACM SIGKDD international conference on knowledge discovery and data mining (KDD-2008). pp. 614\u2013622","DOI":"10.1145\/1401890.1401965"},{"issue":"2","key":"306_CR37","doi-asserted-by":"crossref","first-page":"246","DOI":"10.1109\/TIT.1980.1056150","volume":"26","author":"BW Silverman","year":"1980","unstructured":"Silverman BW (1980) Some asymptotic properties of the probabilistic teacher. IEEE Trans Inf Theory 26(2):246\u2013249","journal-title":"IEEE Trans Inf Theory"},{"key":"306_CR38","volume-title":"Computational learning theory and natural learning systems, vol. III: selecting good models","author":"P Smyth","year":"1995","unstructured":"Smyth P (1995) Learning with probabilistic supervision. In: Petsche T (ed) Computational learning theory and natural learning systems, vol. III: selecting good models. MIT Press, Cambridge"},{"issue":"12","key":"306_CR39","doi-asserted-by":"crossref","first-page":"1253","DOI":"10.1016\/0167-8655(96)00105-5","volume":"17","author":"P Smyth","year":"1996","unstructured":"Smyth P (1996) Bounds on the mean classification error rate of multiple experts. Pattern Recognit Lett 17(12):1253\u20131257","journal-title":"Pattern Recognit Lett"},{"key":"306_CR40","unstructured":"Smyth P, Burl MC, Fayyad UM, Perona P (1994a) Knowledge discovery in large image databases: Dealing with uncertainties in ground truth. In: Knowledge discovery in databases: papers from the 1994 AAAI, workshop (KDD-94). pp 109\u2013120"},{"key":"306_CR41","unstructured":"Smyth P, Fayyad UM, Burl MC, Perona P, Baldi P (1994b) Inferring ground truth from subjective labelling of Venus images. In: Advances in neural information processing systems 7 (NIPS 1994). pp 1085\u20131092"},{"key":"306_CR42","doi-asserted-by":"crossref","unstructured":"Snow R, O\u2019Connor B, Jurafsky D, Ng AY (2008) Cheap and fast\u2013but is it good? Evaluating non-expert annotations for natural language tasks. In: Proceedings of the conference on empirical methods in natural language processing (EMNLP\u201908). pp 254\u2013263","DOI":"10.3115\/1613715.1613751"},{"issue":"3","key":"306_CR43","doi-asserted-by":"crossref","first-page":"659","DOI":"10.1109\/TKDE.2002.1000348","volume":"14","author":"KM Ting","year":"2002","unstructured":"Ting KM (2002) An instance-weighting method to induce cost-sensitive trees. IEEE Trans Knowl Data Eng 14(3):659\u2013665","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"306_CR44","doi-asserted-by":"crossref","first-page":"369","DOI":"10.1613\/jair.120","volume":"2","author":"PD Turney","year":"1995","unstructured":"Turney PD (1995) Cost-sensitive classification: empirical evaluation of a hybrid genetic decision tree induction algorithm. J Artif Intell Res 2:369\u2013409","journal-title":"J Artif Intell Res"},{"key":"306_CR45","unstructured":"Turney PD (2000) Types of cost in inductive concept learning. In: Proceedings of the ICML-2000 workshop on cost-sensitive, learning. pp 15\u201321"},{"key":"306_CR46","doi-asserted-by":"crossref","unstructured":"Verbaeten S, Assche AV (2003) Ensemble methods for noise elimination in classification problems. In: Fourth international workshop on multiple classifier systems. Springer, pp 317\u2013325","DOI":"10.1007\/3-540-44938-8_32"},{"key":"306_CR47","unstructured":"von Ahn L, Dabbish L (2004) Labeling images with a computer game. In: Proceedings of the 2004 conference on human factors in computing systems (CHI 2004). pp 319\u2013326"},{"key":"306_CR48","doi-asserted-by":"crossref","first-page":"315","DOI":"10.1613\/jair.1199","volume":"19","author":"GM Weiss","year":"2003","unstructured":"Weiss GM, Provost FJ (2003) Learning when training data are costly: the effect of class distribution on tree induction. J Artif Intell Res 19:315\u2013354","journal-title":"J Artif Intell Res"},{"key":"306_CR49","unstructured":"Whitehill J, Ruvolo P, fan Wu T, Bergsma J, Movellan J (2009) Whose vote should count more: Optimal integration of labels from labelers of unknown expertise. In: Advances in neural information processing systems 22 (NIPS 2009). pp 2035\u20132043"},{"issue":"1","key":"306_CR50","doi-asserted-by":"crossref","first-page":"123","DOI":"10.1111\/j.2517-6161.1973.tb00944.x","volume":"35","author":"P Whittle","year":"1973","unstructured":"Whittle P (1973) Some general points in the theory of optimal experimental design. J R Stat Soc Ser B 35(1):123\u2013130","journal-title":"J R Stat Soc Ser B"},{"key":"306_CR51","volume-title":"Data mining: practical machine learning tools and techniques","author":"IH Witten","year":"2005","unstructured":"Witten IH, Frank E (2005) Data mining: practical machine learning tools and techniques, 2nd edn. Morgan Kaufmann Publishing, San Francisco","edition":"2"},{"key":"306_CR52","doi-asserted-by":"crossref","unstructured":"Zadrozny B, Langford J, Abe N (2003) Cost-sensitive learning by cost-proportionate example weighting. In: Proceedings of the 3th IEEE international conference on data mining (ICDM 2003). pp 435\u2013442","DOI":"10.1109\/ICDM.2003.1250950"},{"issue":"5","key":"306_CR53","doi-asserted-by":"crossref","first-page":"697","DOI":"10.1287\/mnsc.1050.0488","volume":"52","author":"Z Zheng","year":"2006","unstructured":"Zheng Z, Padmanabhan B (2006) Selectively acquiring customer information: a new data acquisition problem and an active learning-based solution. Manag Sci 52(5):697\u2013712","journal-title":"Manag Sci"},{"issue":"11","key":"306_CR54","doi-asserted-by":"crossref","first-page":"1542","DOI":"10.1109\/TKDE.2005.176","volume":"17","author":"X Zhu","year":"2005","unstructured":"Zhu X, Wu X (2005) Cost-constrained data acquisition for intelligent data preparation. IEEE Trans Knowl Data Eng 17(11):1542\u20131556","journal-title":"IEEE Trans Knowl Data Eng"}],"container-title":["Data Mining and Knowledge Discovery"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10618-013-0306-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10618-013-0306-1\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10618-013-0306-1","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,5,7]],"date-time":"2024-05-07T13:59:09Z","timestamp":1715090349000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10618-013-0306-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,3,16]]},"references-count":54,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2014,3]]}},"alternative-id":["306"],"URL":"https:\/\/doi.org\/10.1007\/s10618-013-0306-1","relation":{},"ISSN":["1384-5810","1573-756X"],"issn-type":[{"value":"1384-5810","type":"print"},{"value":"1573-756X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2013,3,16]]}}}