{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,25]],"date-time":"2025-03-25T20:20:05Z","timestamp":1742934005333,"version":"3.40.3"},"publisher-location":"Cham","reference-count":29,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783319687643"},{"type":"electronic","value":"9783319687650"}],"license":[{"start":{"date-parts":[[2017,1,1]],"date-time":"2017-01-01T00:00:00Z","timestamp":1483228800000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2017]]},"DOI":"10.1007\/978-3-319-68765-0_22","type":"book-chapter","created":{"date-parts":[[2017,10,3]],"date-time":"2017-10-03T03:48:52Z","timestamp":1507002532000},"page":"260-271","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["The Combination of Decision in Crowds When the Number of Reliable Annotator Is Scarce"],"prefix":"10.1007","author":[{"given":"Agus Budi","family":"Raharjo","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mohamed","family":"Quafafou","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2017,10,4]]},"reference":[{"issue":"2","key":"22_CR1","doi-asserted-by":"publisher","first-page":"189","DOI":"10.1177\/0165551512437638","volume":"38","author":"E Estell\u00e9s-Arolas","year":"2012","unstructured":"Estell\u00e9s-Arolas, E., Gonz\u00e1lez-Ladr\u00f3n-De-Guevara, F.: Towards an integrated crowdsourcing definition. J. Inf. Sci. 38(2), 189\u2013200 (2012)","journal-title":"J. Inf. Sci."},{"key":"22_CR2","unstructured":"Howe, J.: Crowdsourcing: How the Power of the Crowd is Driving the Future of Business. Business books. Random House Business (2008)"},{"issue":"14","key":"22_CR3","doi-asserted-by":"publisher","first-page":"6190","DOI":"10.1016\/j.eswa.2014.04.012","volume":"41","author":"A Tarasov","year":"2014","unstructured":"Tarasov, A., Delany, S.J., Namee, B.M.: Dynamic estimation of worker reliability in crowdsourcing for regression tasks: making it work. Expert Syst. Appl. 41(14), 6190\u20136210 (2014)","journal-title":"Expert Syst. Appl."},{"key":"22_CR4","doi-asserted-by":"crossref","unstructured":"Heer, J., Bostock, M.: Crowdsourcing graphical perception: using mechanical turk to assess visualization design. In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems, CHI 2010, pp. 203\u2013212. ACM, New York (2010)","DOI":"10.1145\/1753326.1753357"},{"key":"22_CR5","unstructured":"Ho, C.J., Jabbari, S., Vaughan, J.W.: Adaptive task assignment for crowdsourced classification. In: Proceedings of the 30th International Conference on International Conference on Machine Learning, ICML2013, vol. 28, pp. I-534-I-542, JMLR.org (2013)"},{"key":"22_CR6","doi-asserted-by":"crossref","unstructured":"Boutsis, I., Kalogeraki, V.: On task assignment for real-time reliable crowdsourcing. In: 2014 IEEE 34th International Conference on Distributed Computing Systems, pp. 1\u201310, June 2014","DOI":"10.1109\/ICDCS.2014.9"},{"key":"22_CR7","doi-asserted-by":"crossref","unstructured":"Moayedikia, A., Ong, K.L., Boo, Y.L., Yeoh, W.: Bee colony based worker reliability estimation algorithm in microtask crowdsourcing. In: 2016 15th IEEE International Conference on Machine Learning and Applications (ICMLA), pp. 713\u2013717, December 2016","DOI":"10.1109\/ICMLA.2016.0127"},{"key":"22_CR8","first-page":"2655","volume":"13(Sep)","author":"O Dekel","year":"2012","unstructured":"Dekel, O., Gentile, C., Sridharan, K.: Selective sampling and active learning from single and multiple teachers. J. Mach. Learn. Res. 13(Sep), 2655\u20132697 (2012)","journal-title":"J. Mach. Learn. Res."},{"key":"22_CR9","doi-asserted-by":"crossref","unstructured":"Downs, J.S., Holbrook, M.B., Sheng, S., Cranor, L.F.: Are your participants gaming the system? Screening mechanical turk workers. In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems, CHI 2010, pp. 2399\u20132402. ACM, New York (2010)","DOI":"10.1145\/1753326.1753688"},{"issue":"1","key":"22_CR10","first-page":"491","volume":"13","author":"VC Raykar","year":"2012","unstructured":"Raykar, V.C., Yu, S.: Eliminating spammers and ranking annotators for crowdsourced labeling tasks. J. Mach. Learn. Res. 13(1), 491\u2013518 (2012)","journal-title":"J. Mach. Learn. Res."},{"issue":"3","key":"22_CR11","doi-asserted-by":"publisher","first-page":"326","DOI":"10.1002\/int.21702","volume":"30","author":"J Hern\u00e1ndez-Gonz\u00e1lez","year":"2015","unstructured":"Hern\u00e1ndez-Gonz\u00e1lez, J., Inza, I., Lozano, J.A.: Multidimensional learning from crowds: usefulness and application of expertise detection. Int. J. Intell. Syst. 30(3), 326\u2013354 (2015)","journal-title":"Int. J. Intell. Syst."},{"key":"22_CR12","doi-asserted-by":"publisher","first-page":"254","DOI":"10.1016\/j.ins.2016.12.026","volume":"382\u2013383","author":"J Zhang","year":"2017","unstructured":"Zhang, J., Sheng, V.S., Li, Q., Wu, J., Wu, X.: Consensus algorithms for biased labeling in crowdsourcing. Inf. Sci. 382\u2013383, 254\u2013273 (2017)","journal-title":"Inf. Sci."},{"issue":"1","key":"22_CR13","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1111\/j.2517-6161.1977.tb01600.x","volume":"39","author":"AP Dempster","year":"1977","unstructured":"Dempster, A.P., Laird, N.M., Rubin, D.B.: Maximum likelihood from incomplete data via the em algorithm. J. Roy. Stat. Soc. B 39(1), 1\u201338 (1977)","journal-title":"J. Roy. Stat. Soc. B"},{"key":"22_CR14","unstructured":"Condorcet, M.d.: Essai sur l\u2019application de l\u2019analyse \u00e0 la probabilit\u00e9 des d\u00e9cisions rendues \u00e0 la pluralit\u00e9 des voix (1785)"},{"key":"22_CR15","unstructured":"Lichman, M.: UCI machine learning repository (2013)"},{"key":"22_CR16","unstructured":"Whitehill, J., Ruvolo, P., Wu, T., Bergsma, J., Movellan, J.: Whose vote should count more: Optimal integration of labels from labelers of unknown expertise. In: Proceedings of the 22nd International Conference on Neural Information Processing Systems, NIPS 2009, USA, Curran Associates Inc., pp. 2035\u20132043 (2009)"},{"key":"22_CR17","doi-asserted-by":"crossref","unstructured":"Welinder, P., Perona, P.: Online crowdsourcing: Rating annotators and obtaining cost-effective labels. In: 2010 IEEE Computer Society Conference on Computer Vision and Pattern Recognition - Workshops, pp. 25\u201332, June 2010","DOI":"10.1109\/CVPRW.2010.5543189"},{"key":"22_CR18","first-page":"1297","volume":"11","author":"VC Raykar","year":"2010","unstructured":"Raykar, V.C., Yu, S., Zhao, L.H., Valadez, G.H., Florin, C., Bogoni, L., Moy, L.: Learning from crowds. J. Mach. Learn. Res. 11, 1297\u20131322 (2010)","journal-title":"J. Mach. Learn. Res."},{"key":"22_CR19","unstructured":"Yan, Y., Fung, G., Schmidt, M., Hermosillo, G., Bogoni, L., Moy, L., Dy, J.G.: Modeling annotator expertise: learning when everyone knows a bit of something. In: In Proceedings of the Thirteenth International Conference on Artificial Intelligence and Statistics (AISTATS, 2010), pp. 932\u2013939 (2010)"},{"key":"22_CR20","doi-asserted-by":"crossref","unstructured":"Wolley, C., Quafafou, M.: Scalable experts selection when learning from noisy labelers. In: 12th International Conference on Machine Learning and Applications (ICMLA) Poster Session (2013)","DOI":"10.1109\/ICMLA.2013.81"},{"key":"22_CR21","doi-asserted-by":"crossref","unstructured":"Zighed, D.A., Ritschard, G., Marcellin, S.: Asymmetric and sample size sensitive entropy measures for supervised learning. In: Ras Z.W., Tsay L.S. (eds.) Advances in Intelligent Information Systems. Studies in Computational Intelligence, vol 265, pp. 27\u201342. Springer, Heidelberg (2010)","DOI":"10.1007\/978-3-642-05183-8_2"},{"issue":"1","key":"22_CR22","doi-asserted-by":"publisher","first-page":"91","DOI":"10.1007\/s00355-011-0546-1","volume":"39","author":"B Peleg","year":"2012","unstructured":"Peleg, B., Zamir, S.: Extending the condorcet jury theorem to a general dependent jury. Soc. Choice Welfare 39(1), 91\u2013125 (2012)","journal-title":"Soc. Choice Welfare"},{"issue":"1","key":"22_CR23","doi-asserted-by":"publisher","first-page":"6","DOI":"10.1186\/s12880-015-0049-0","volume":"15","author":"K Gottlieb","year":"2015","unstructured":"Gottlieb, K., Hussain, F.: Voting for image scoring and assessment (visa) - theory and application of a 2 + 1 reader algorithm to improve accuracy of imaging endpoints in clinical trials. BMC Med. Imaging 15(1), 6 (2015)","journal-title":"BMC Med. Imaging"},{"key":"22_CR24","unstructured":"Xia, L.: Quantitative extensions of the condorcet jury theorem with strategic agents. In: Proceedings of the Thirtieth AAAI Conference on Artificial Intelligence, February 12\u201317, 2016, Phoenix, Arizona, USA, pp. 644\u2013650 (2016)"},{"key":"22_CR25","unstructured":"Jain, B.J.: Condorcet\u2019s jury theorem for consensus clustering. CoRR abs\/1604.07711 (2016)"},{"issue":"2","key":"22_CR26","doi-asserted-by":"publisher","first-page":"171","DOI":"10.1023\/A:1015551010381","volume":"52","author":"WV Gehrlein","year":"2002","unstructured":"Gehrlein, W.V.: Condorcet\u2019s paradox and the likelihood of its occurrence: different perspectives on balanced preferences*. Theor. Decis. 52(2), 171\u2013199 (2002)","journal-title":"Theor. Decis."},{"key":"22_CR27","unstructured":"Peyton, H.: Group choice and individual judgements, pp. 181\u2013200. Cambridge University Press, Cambridge (1997)"},{"issue":"1","key":"22_CR28","first-page":"20","volume":"28","author":"AP Dawid","year":"1979","unstructured":"Dawid, A.P.: A.M.S.: Maximum likelihood estimation of observer error-rates using the em algorithm. J. Roy. Stat. Soc.: Ser. C (Appl. Stat.) 28(1), 20\u201328 (1979)","journal-title":"J. Roy. Stat. Soc.: Ser. C (Appl. Stat.)"},{"issue":"3(Part 2)","key":"22_CR29","doi-asserted-by":"publisher","first-page":"5866","DOI":"10.1016\/j.eswa.2008.07.018","volume":"36","author":"IC Yeh","year":"2009","unstructured":"Yeh, I.C., Yang, K.J., Ting, T.M.: Knowledge discovery on RFM model using Bernoulli sequence. Expert Syst. Appl. 36(3(Part 2)), 5866\u20135871 (2009)","journal-title":"Expert Syst. Appl."}],"container-title":["Lecture Notes in Computer Science","Advances in Intelligent Data Analysis XVI"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-319-68765-0_22","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,6,27]],"date-time":"2024-06-27T12:53:28Z","timestamp":1719492808000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-319-68765-0_22"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017]]},"ISBN":["9783319687643","9783319687650"],"references-count":29,"URL":"https:\/\/doi.org\/10.1007\/978-3-319-68765-0_22","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2017]]},"assertion":[{"value":"4 October 2017","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"IDA","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Symposium on Intelligent Data Analysis","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"London","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"United Kingdom","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2017","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"26 October 2017","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"28 October 2017","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"16","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ida2017","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/www.dcs.bbk.ac.uk\/ida2017\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}