{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,26]],"date-time":"2025-12-26T22:33:23Z","timestamp":1766788403246},"reference-count":72,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2014,10,5]],"date-time":"2014-10-05T00:00:00Z","timestamp":1412467200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Knowl Inf Syst"],"published-print":{"date-parts":[[2015,11]]},"DOI":"10.1007\/s10115-014-0790-7","type":"journal-article","created":{"date-parts":[[2014,10,4]],"date-time":"2014-10-04T11:02:09Z","timestamp":1412420529000},"page":"271-294","update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":22,"title":["A unified statistical framework for crowd labeling"],"prefix":"10.1007","volume":"45","author":[{"given":"Jafar","family":"Muhammadi","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hamid R.","family":"Rabiee","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Abbas","family":"Hosseini","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2014,10,5]]},"reference":[{"issue":"2","key":"790_CR1","doi-asserted-by":"crossref","first-page":"427","DOI":"10.1111\/j.0006-341X.2004.00187.x","volume":"60","author":"PS Albert","year":"2004","unstructured":"Albert PS, Dodd LE (2004) A cautionary note on the robustness of latent class models for estimating diagnostic error without a gold standard. Biometrics 60(2):427\u2013435","journal-title":"Biometrics"},{"key":"790_CR2","doi-asserted-by":"crossref","unstructured":"Ali SM, Silvey SD (1966) A general class of coefficients of divergence of one distribution from another. J R Stat Soc Ser B (Methodological) 28:131\u2013142","DOI":"10.1111\/j.2517-6161.1966.tb00626.x"},{"key":"790_CR3","doi-asserted-by":"crossref","unstructured":"Attenberg J, Provost FJ (2011) Online active inference and learning, KDD, pp 186\u2013194","DOI":"10.1145\/2020408.2020443"},{"key":"790_CR4","unstructured":"Bachrach Y, Graepel T, Minka T, Guiver J (2012) How To grade a test without knowing the answers\u2014a Bayesian graphical model for adaptive crowdsourcing and aptitude testing, ICML"},{"issue":"1","key":"790_CR5","doi-asserted-by":"crossref","first-page":"61","DOI":"10.1007\/s10115-012-0571-0","volume":"37","author":"A Ballatore","year":"2013","unstructured":"Ballatore A, Bertolotto M, Wilson DC (2013) Geographic knowledge extraction and semantic similarity in OpenStreetMap. Knowl Inf Syst 37(1):61\u201381","journal-title":"Knowl Inf Syst"},{"key":"790_CR6","doi-asserted-by":"crossref","unstructured":"Bernstein MS, Little G, Miller RC, Hartmann B, Ackerman MS, Karger DR, Crowell D, Panovich K (2010) Soylent: a word processor with a crowd inside. In: Proceedings of the 23rd annual ACM symposium on user interface software and technology, UIST \u201910. ACM, New York, pp 313\u2013322","DOI":"10.1145\/1866029.1866078"},{"key":"790_CR7","unstructured":"Bernstein MS, Karger DR, Miller RC, Brandt J (2012) Analytic methods for optimizing realtime crowdsourcing. CoRR"},{"key":"790_CR8","doi-asserted-by":"crossref","first-page":"13","DOI":"10.1177\/109434209200600103","volume":"6","author":"MW Berry","year":"1992","unstructured":"Berry MW (1992) Large scale sparse singular value computations. Int J Supercomput Appl 6:13\u201349","journal-title":"Int J Supercomput Appl"},{"key":"790_CR9","first-page":"740","volume-title":"Pattern recognition and machine learning","author":"CM Bishop","year":"2006","unstructured":"Bishop CM (2006) Pattern recognition and machine learning, vol 1. Springer, New York, p 740"},{"key":"790_CR10","doi-asserted-by":"crossref","unstructured":"Branson S, Wah C, Babenko B, Schroff F, Welinder P, Perona P, Belongie S (2010) Visual recognition with humans in the loop, European Conference on Computer Vision (ECCV)","DOI":"10.1007\/978-3-642-15561-1_32"},{"key":"790_CR11","unstructured":"Brew A, Greene D, Cunningham P (2010) The interaction between supervised learning and crowdsourcing, NIPS workshop on computational social science and the Wisdom of Crowds"},{"key":"790_CR12","first-page":"64","volume":"2013","author":"X Chen","year":"2013","unstructured":"Chen X, Lin Q, Zhou D (2013) Optimistic knowledge gradient policy for optimal budget allocation in crowdsourcing. ICML 2013:64\u201372","journal-title":"ICML"},{"key":"790_CR13","doi-asserted-by":"crossref","unstructured":"Chilton LB, Horton JJ, Miller RC, Azenkot S (2010) Task search in a human computation market. In: Proceedings of the ACM SIGKDD workshop on human computation (HCOMP \u201910). ACM, New York","DOI":"10.1145\/1837885.1837889"},{"key":"790_CR14","doi-asserted-by":"crossref","unstructured":"Dai P, Mausam, Weld DS (2011) Artificial intelligence for artificial intelligence, AAAI","DOI":"10.1609\/aaai.v25i1.8096"},{"issue":"1","key":"790_CR15","first-page":"20","volume":"28","author":"AP Dawid","year":"1979","unstructured":"Dawid AP, Skene AM (1979) Maximum likelihood estimation of observer error-rates using the EM algorithm. J R Stat Soc Ser C 28(1):20\u201328","journal-title":"J R Stat Soc Ser C"},{"key":"790_CR16","doi-asserted-by":"crossref","unstructured":"Dekel O, Shamir O (2009a) Good learners for evil teachers. In: ICML, vol 382, 30. ACM, New York","DOI":"10.1145\/1553374.1553404"},{"key":"790_CR17","unstructured":"Dekel O, Shamir O (2009b) Vox populi, collecting high-quality labels from a crowd. In: COLT"},{"key":"790_CR18","volume-title":"Mind over machine\u2014the power of human intuition and expertise in the era of the computer","author":"HL Dreyfus","year":"1988","unstructured":"Dreyfus HL, Dreyfus SE (1988) Mind over machine\u2014the power of human intuition and expertise in the era of the computer. Free Press, New York"},{"key":"790_CR19","doi-asserted-by":"crossref","unstructured":"Eagle N (2009) txteagle: mobile crowdsourcing. In: Proceedings of the 3rd international conference on internationalization, design and global development: held as part of HCI international 2009, IDGD \u201909. Springer, Berlin, pp 447\u2013456","DOI":"10.1007\/978-3-642-02767-3_50"},{"key":"790_CR20","doi-asserted-by":"crossref","unstructured":"Fang M, Zhu X, Li B, Ding W, Wu X (2012) Self-taught active learning from crowds. ICDM, pp 858\u2013863","DOI":"10.1109\/ICDM.2012.64"},{"key":"790_CR21","doi-asserted-by":"crossref","unstructured":"Frank E, Hall M (2001) A simple approach to ordinal classification. In: ECML \u201901: proceedings of the 12th European conference on machine learning. Springer, London, pp 145\u2013156","DOI":"10.1007\/3-540-44795-4_13"},{"issue":"2","key":"790_CR22","doi-asserted-by":"crossref","first-page":"249","DOI":"10.1007\/s10115-012-0507-8","volume":"35","author":"Y Fu","year":"2013","unstructured":"Fu Y, Zhu X, Li B (2013) A survey on instance selection for active learning. Knowl Inf Syst 35(2):249\u2013283","journal-title":"Knowl Inf Syst"},{"key":"790_CR23","unstructured":"Grady C, Lease M (2010) Crowdsourcing document relevance assessment with mechanical turk. In: Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with Amazon\u2019s mechanical turk, Association for Computational Linguistics, pp 172\u2013179"},{"key":"790_CR24","unstructured":"Howe J (2008) Crowdsourcing: why the power of the crowd is driving the future of business, 1st edn. Crown Business. United States"},{"key":"790_CR25","unstructured":"Ho CJ, Vaughan JW (2012) Online task assignment in crowdsourcing markets, AAAI"},{"issue":"1","key":"790_CR26","first-page":"534","volume":"28","author":"CJ Ho","year":"2013","unstructured":"Ho CJ, Jabbari S, Vaughan JW (2013) Adaptive task assignment for crowdsourced classification. JMLR W&CP 28(1):534\u2013542","journal-title":"JMLR W&CP"},{"key":"790_CR27","unstructured":"Ipeirotis PG, Provost F, Sheng V, Wang J (2010) Repeated labeling using multiple noisy labelers. In CeDER working papers"},{"key":"790_CR28","doi-asserted-by":"crossref","unstructured":"Ipeirotis PG, Provost F, Wang J (2010) Quality management on amazon mechanical turk. In: Proceedings of the ACM SIGKDD workshop on human computation (HCOMP \u201910), pp 64\u201367. ACM, New York","DOI":"10.1145\/1837885.1837906"},{"key":"790_CR29","unstructured":"Janssens JHM (2010) Ranking images on semantic attributes using human computation. In: NIPS workshop on computational social science and the Wisdom of crowds"},{"key":"790_CR30","doi-asserted-by":"crossref","first-page":"1095","DOI":"10.1145\/2348283.2348486","volume":"2012","author":"HJ Jung","year":"2012","unstructured":"Jung HJ, Lease M (2012) Inferring missing relevance judgments from crowd workers via probabilistic matrix factorization. SIGIR 2012:1095\u20131096","journal-title":"SIGIR"},{"key":"790_CR31","unstructured":"Jung HJ, Lease M (2012) Improving quality of crowdsourced labels via probabilistic matrix factorization. In: Proceedings of the 4th human computation workshop (HCOMP) at AAAI"},{"key":"790_CR32","unstructured":"Kajino H, Tsuboi Y, Sato I, Kashima H (2012) Learning from crowds and experts. In: AAAI human computation technical reports WS-12-08"},{"key":"790_CR33","doi-asserted-by":"crossref","unstructured":"Kajino H, Tsuboi Y, Kashima H (2012) A convex formulation for learning from crowds. In AAAI","DOI":"10.1527\/tjsai.27.133"},{"key":"790_CR34","unstructured":"Karger DR, Oh S, Shah D (2011a) Budget-optimal task allocation for reliable crowdsourcing systems. CoRR abs\/1110. 3564"},{"key":"790_CR35","doi-asserted-by":"crossref","unstructured":"Karger DR, Oh S, Shah D (2011b) Budget-optimal crowdsourcing using low-rank matrix approximations. In: 49th annual conference on communication, control, and computing (Allerton), pp 284\u2013291","DOI":"10.1109\/Allerton.2011.6120180"},{"key":"790_CR36","volume-title":"Human computation, synthesis lectures on artificial intelligence and machine learning","author":"E Law","year":"2011","unstructured":"Law E, von Ahn L (2011) Human computation, synthesis lectures on artificial intelligence and machine learning. Morgan & Claypool Publishers, Los Altos"},{"key":"790_CR37","doi-asserted-by":"crossref","unstructured":"Little G, Chilton LB, Goldman M, Miller RC (2009) TurKit: tools for iterative tasks on mechanical turk. In: Proceedings of the ACM SIGKDD workshop on human computation, HCOMP \u201909. ACM, Paris, pp 29\u201330","DOI":"10.1145\/1600150.1600159"},{"key":"790_CR38","doi-asserted-by":"crossref","unstructured":"Little G, Chilton LB, Goldman M, Miller RC (2010) Exploring iterative and parallel human computation processes. In: Proceedings of the ACM SIGKDD workshop on human computation, HCOMP \u201910. ACM, New York, pp 68\u201376","DOI":"10.1145\/1837885.1837907"},{"key":"790_CR39","unstructured":"Liu C, Wang YM (2012) TrueLabel + confusions: a spectrum of probabilistic models in analyzing multiple ratings. ICML"},{"key":"790_CR40","unstructured":"Liu Q, Peng J, Ihler A (2012) Variational inference for crowdsourcing. In: Advances in neural information processing systems (NIPS), pp 701\u2013709"},{"key":"790_CR41","doi-asserted-by":"crossref","unstructured":"Mason WA, Watts DJ (2009) Financial incentives and the \u201cperformance of crowds\u201d. KDD, pp 100\u2013108","DOI":"10.1145\/1600150.1600175"},{"key":"790_CR42","unstructured":"Mason W, Suri S (2010) Conducting behavioral research on amazon\u2019s mechanical turk. Social Science Research Network Working Paper Series"},{"issue":"18","key":"790_CR43","doi-asserted-by":"crossref","first-page":"1174","DOI":"10.1016\/j.artint.2007.10.009","volume":"171","author":"J Mccarthy","year":"2007","unstructured":"Mccarthy J (2007) From here to human-level AI. Artif Intell 171(18):1174\u20131182","journal-title":"Artif Intell"},{"issue":"6","key":"790_CR44","doi-asserted-by":"crossref","first-page":"1087","DOI":"10.1063\/1.1699114","volume":"21","author":"N Metropolis","year":"2004","unstructured":"Metropolis N, Rosenbluth AW, Rosenbluth MN, Teller AH, Teller E (2004) Equation of state calculations by fast computing machines. J Chem Phys 21(6):1087\u20131092","journal-title":"J Chem Phys"},{"key":"790_CR45","unstructured":"Minka TP (2001) Expectation propagation for approximate Bayesian inference. In: Proceedings of the seventeenth conference on uncertainty in artificial intelligence. Morgan Kaufmann Publishers Inc., Massachusetts, pp 362\u2013369"},{"key":"790_CR46","unstructured":"Minka TP (2001) A family of algorithms for approximate Bayesian inference (Doctoral dissertation, Massachusetts Institute of Technology)"},{"key":"790_CR47","unstructured":"Minsky ML (1992) Future of AI technology. Toshiba Rev 47(7). https:\/\/web.media.mit.edu\/~minsky\/papers\/CausalDiversity.html"},{"key":"790_CR48","unstructured":"Paquet U, Gael JV, Stern D, Kasneci G, Herbrich R, Graepel T (2010) Vuvuzelas active learning for online classification. In: NIPS workshop on computational social science and the Wisdom of Crowds"},{"key":"790_CR49","volume-title":"Probabilistic reasoning in intelligent systems: networks of plausible inference","author":"J Pearl","year":"1988","unstructured":"Pearl J (1988) Probabilistic reasoning in intelligent systems: networks of plausible inference, 2nd edn. Morgan Kaufmann, San Francisco","edition":"2"},{"key":"790_CR50","doi-asserted-by":"crossref","unstructured":"Potter A, McClure M, Sellers K (2010) Mass collaboration problem solving: a new approach to wicked problems. In: 2010 International symposium on collaborative technologies and systems, pp 398\u2013407","DOI":"10.1109\/CTS.2010.5478486"},{"key":"790_CR51","doi-asserted-by":"crossref","first-page":"797","DOI":"10.2307\/2533043","volume":"52","author":"Y Qu","year":"1996","unstructured":"Qu Y, Tan M, Kutner MH (1996) Random effects models in latent class analysis for evaluating accuracy of diagnostic tests. Biometrics 52:797\u2013810","journal-title":"Biometrics"},{"issue":"1","key":"790_CR52","first-page":"15","volume":"9","author":"MJ Raddick","year":"2009","unstructured":"Raddick MJ, Bracey G, Gay PL, Lintott CJ, Murray P, Schawinski K, Szalay AS et al (2009) Galaxy Zoo: exploring the motivations of citizen science volunteers. Astron Educ Rev 9(1):15","journal-title":"Astron Educ Rev"},{"key":"790_CR53","doi-asserted-by":"crossref","unstructured":"Raykar VC, Yu S, Zhao LH, Jerebko AK, Florin C, Valadez GH, Bogoni L, Moy L (2009) Supervised learning from multiple experts: whom to trust when everyone lies a bit. In: ICML, p 112","DOI":"10.1145\/1553374.1553488"},{"key":"790_CR54","first-page":"1297","volume":"99","author":"VC Raykar","year":"2010","unstructured":"Raykar VC, Yu S, Zhao LH, Valadez GH, Florin C, Bogoni L, Moy L (2010) Learning from crowds. J Mach Learn Resour 99:1297\u20131322","journal-title":"J Mach Learn Resour"},{"key":"790_CR55","first-page":"491","volume":"13","author":"VC Raykar","year":"2012","unstructured":"Raykar VC, Yu S (2012) Eliminating spammers and ranking annotators for crowdsourced labeling tasks. J Mach Learn Res 13:491\u2013518","journal-title":"J Mach Learn Res"},{"key":"790_CR56","unstructured":"Ruvolo P, Whitehill J, Movel-lan JR (2010) Exploiting structure in crowdsourcing tasks via latent factor models. Technical Report TR2010. 01, Machine Perception Laboratory"},{"key":"790_CR57","unstructured":"Salakhutdinov R, Mnih A (2008) Probabilistic matrix factorization. In NIPS 2008, vol 20"},{"key":"790_CR58","unstructured":"Seeger M (2006) Bayesian modeling in machine learning: a tutorial review, Technical Report EPFL-161462"},{"key":"790_CR59","doi-asserted-by":"crossref","unstructured":"Shen W, Campbell JP, Straub D, Schwartz R (2011) Assessing the speaker recognition performance of naive listeners using mechanical turk. ICASSP 2011, pp 5916\u20135919","DOI":"10.1109\/ICASSP.2011.5947708"},{"key":"790_CR60","doi-asserted-by":"crossref","unstructured":"Sheng VS, Provost F, Ipeirotis PG (2008) Get another label? Improving data quality and data mining using multiple, noisy labelers. In: Proceeding of the 14th ACM SIGKDD international conference on Knowledge discovery and data mining, KDD \u201908. ACM, New York, pp 614\u2013622","DOI":"10.1145\/1401890.1401965"},{"key":"790_CR61","unstructured":"Smyth P, Fayyad U, Burl M, Perona P, Baldi P (1997) Inferring ground truth from subjective labeling of venus images. In: NIPS, pp 1\u20139"},{"key":"790_CR62","doi-asserted-by":"crossref","unstructured":"Snow R, O\u2019Connor, Jurafsky D, Ng A (2008) Cheap and Fast\u2014but is it good? Evaluating non-expert annotations for natural language tasks. In: Proceedings of EMNLP-08","DOI":"10.3115\/1613715.1613751"},{"key":"790_CR63","unstructured":"Srebro N, Jaakkola T (2003) Weighted low-rank approximations. In: Proceedings of the Twentieth international conference of machine learning (ICML 2003). AAAI Press, pp 720\u2013727"},{"key":"790_CR64","doi-asserted-by":"crossref","unstructured":"von Ahn L, Dabbish L (2004) Labeling images with a computer game. In: Proceedings of the SIGCHI conference on human factors in computing systems, CHI \u201904. ACM, Vienna, pp 319\u2013326","DOI":"10.1145\/985692.985733"},{"key":"790_CR65","unstructured":"Wais P, Lingamneni S, Cook D, Fennell J, Goldenberg B, Lubarov D, Marin D, Simons H (2010) Towards building a high-quality workforce with mechanical turk. In: NIPS workshop on computational social science and the Wisdom of Crowds"},{"issue":"3","key":"790_CR66","doi-asserted-by":"crossref","first-page":"66","DOI":"10.1109\/MIS.2006.46","volume":"21","author":"DL Waltz","year":"2006","unstructured":"Waltz DL (2006) Evolution, sociobiology, and the future of artificial intelligence. IEEE Intell Syst 21(3):66\u201369","journal-title":"IEEE Intell Syst"},{"key":"790_CR67","first-page":"2424","volume":"23","author":"P Welinder","year":"2010","unstructured":"Welinder P, Branson S, Belongie S, Perona P (2010) The multidimensional Wisdom of crowds. Adv Neural Inf Process Syst 23:2424\u20132432","journal-title":"Adv Neural Inf Process Syst"},{"key":"790_CR68","unstructured":"Whitehill J, Ruvolo P, Fan Wu T, Bergsma J, Movellan J (2009) Whose vote should count more: optimal integration of labels from labelers of unknown expertise. Adv Neural Inf Process Syst 22:2035\u20132043"},{"key":"790_CR69","unstructured":"Yan Y, Rosales R, Fung G, Schmidt MW, Valadez GH, Bogoni L, Moy L, Dy JG (2010) Modeling annotator expertise: learning when everybody knows a bit of something. J Mach Learn Res 9:932\u2013939"},{"key":"790_CR70","doi-asserted-by":"crossref","first-page":"433","DOI":"10.1007\/s10915-013-9682-3","volume":"56","author":"M Yan","year":"2013","unstructured":"Yan M, Yang Y, Osher S (2013) Exact low-rank matrix completion from sparsely corrupted entries via adaptive outlier pursuit. J Sci Comput 56:433\u2013449","journal-title":"J Sci Comput"},{"key":"790_CR71","volume-title":"An introduction to Bayesian inference in econometrics","author":"A Zellner","year":"1971","unstructured":"Zellner A (1971) An introduction to Bayesian inference in econometrics, vol 17. Wiley, London"},{"key":"790_CR72","unstructured":"Zhu D, Carterette B (2010) An analysis of assessor behavior in crowdsourced preference judgments. In: ACM SIGIR workshop on crowdsourcing for search evaluation"}],"container-title":["Knowledge and Information Systems"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-014-0790-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10115-014-0790-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-014-0790-7","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,6,3]],"date-time":"2024-06-03T12:26:55Z","timestamp":1717417615000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10115-014-0790-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014,10,5]]},"references-count":72,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2015,11]]}},"alternative-id":["790"],"URL":"https:\/\/doi.org\/10.1007\/s10115-014-0790-7","relation":{},"ISSN":["0219-1377","0219-3116"],"issn-type":[{"value":"0219-1377","type":"print"},{"value":"0219-3116","type":"electronic"}],"subject":[],"published":{"date-parts":[[2014,10,5]]}}}