{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,22]],"date-time":"2025-11-22T11:04:50Z","timestamp":1763809490477},"publisher-location":"Cham","reference-count":23,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783319261898"},{"type":"electronic","value":"9783319261904"}],"license":[{"start":{"date-parts":[[2015,1,1]],"date-time":"2015-01-01T00:00:00Z","timestamp":1420070400000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2015]]},"DOI":"10.1007\/978-3-319-26190-4_20","type":"book-chapter","created":{"date-parts":[[2015,10,26]],"date-time":"2015-10-26T11:03:05Z","timestamp":1445857385000},"page":"293-308","source":"Crossref","is-referenced-by-count":8,"title":["A Majority of Wrongs Doesn\u2019t Make It Right - On Crowdsourcing Quality for Skewed Domain Tasks"],"prefix":"10.1007","author":[{"given":"Kinda","family":"El Maarry","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ulrich","family":"G\u00fcntzer","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Wolf-Tilo","family":"Balke","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2015,12,25]]},"reference":[{"key":"20_CR1","doi-asserted-by":"crossref","unstructured":"Sorokin, A., Forsyth, D.: Utility data annotation with amazon mechanical turk. In: 2008 IEEE Computer Society Conference on Computer Vision and Pattern Recognition Workshops, CVPR Workshops (2008)","DOI":"10.1109\/CVPRW.2008.4562953"},{"key":"20_CR2","doi-asserted-by":"crossref","unstructured":"Kazai, G., Craswell, N., Yilmaz, E., Tahaghoghi, S.M.: An analysis of systematic judging errors in information retrieval. In: Proceedings of the 21st ACM International Conference on Information and Knowledge Management - CIKM 2012, p. 105 (2012)","DOI":"10.1145\/2396761.2396779"},{"key":"20_CR3","doi-asserted-by":"crossref","unstructured":"Selke, J., Lofi, C., Balke, W.-T.: Pushing the boundaries of crowd-enabled databases with query-driven schema expansion. In: 38th International Conference on Very Large Data Bases (VLDB), pp. 538\u2013549 (2012)","DOI":"10.14778\/2168651.2168655"},{"issue":"1","key":"20_CR4","doi-asserted-by":"publisher","first-page":"109","DOI":"10.1007\/s13222-012-0092-8","volume":"12","author":"C Lofi","year":"2012","unstructured":"Lofi, C., Selke, J., Balke, W.-T.: Information extraction meets crowdsourcing: a promising couple. Datenbank-Spektrum 12(1), 109\u2013120 (2012)","journal-title":"Datenbank-Spektrum"},{"issue":"1","key":"20_CR5","doi-asserted-by":"publisher","first-page":"22","DOI":"10.1007\/s10044-002-0173-7","volume":"6","author":"LI Kuncheva","year":"2003","unstructured":"Kuncheva, L.I., Whitaker, C.J., Shipp, C.A., Duin, R.P.W.: Limits on the majority vote accuracy in classifier fusion. Pattern Anal. Appl. PAA 6(1), 22\u201331 (2003)","journal-title":"Pattern Anal. Appl. PAA"},{"key":"20_CR6","doi-asserted-by":"crossref","unstructured":"Ipeirotis, P.G., Provost, F., Wang, J.: Quality management on amazon mechanical turk. In: Proceedings of the 2nd Human Computation Workshop (HCOMP), pp. 0\u20133 (2010)","DOI":"10.1145\/1837885.1837906"},{"key":"20_CR7","unstructured":"Finin, T., Murnane, W., Karandikar, A., Keller, N., Martineau, J., Dredze, M.: Annotating named entities in twitter data with crowdsourcing. In: Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk, vol. 2010, pp. 80\u201388, Hune (2010)"},{"key":"20_CR8","doi-asserted-by":"crossref","unstructured":"Lofi, C., El Maarry, K., Balke, W.-T.: Skyline queries in crowd-enabled databases. In: International Conference on Extending Database Technology (EDBT), March 2013","DOI":"10.1145\/2452376.2452431"},{"key":"20_CR9","doi-asserted-by":"crossref","unstructured":"Kouloumpis, E., Wilson, T., Moore, J.: Twitter sentiment analysis: the good the bad and the OMG! In: Fifth International AAAI Conference on Weblogs and Social Media, pp. 538\u2013541 (2011)","DOI":"10.1609\/icwsm.v5i1.14185"},{"key":"20_CR10","doi-asserted-by":"crossref","unstructured":"Callison-Burch, C.: Fast, cheap, and creative: evaluating translation quality using amazon\u2019s mechanical turk. In: Proceedings of the 2009 Conference on Empirical Methods in Natural Language Processing EMNLP 2009, vol. 1, issue 1, pp. 286\u2013295 (2009)","DOI":"10.3115\/1699510.1699548"},{"issue":"1","key":"20_CR11","first-page":"20","volume":"28","author":"AP Dawid","year":"1979","unstructured":"Dawid, A.P., Skene, A.M.: Maximum likelihood estimation of observer error-rates using the EM algorithm. J. Roy. Stat. Soc. Ser. C Appl. Stat. JSTOR 28(1), 20\u201328 (1979)","journal-title":"J. Roy. Stat. Soc. Ser. C Appl. Stat. JSTOR"},{"key":"20_CR12","first-page":"1297","volume":"11","author":"VC Raykar","year":"2010","unstructured":"Raykar, V.C., Yu, S., Zhao, L.H., Valadez, G.H., Florin, C., Bogoni, L., Moy, L.: Learning from crowds. J. Mach. Learn. Res. 11, 1297\u20131322 (2010). JMLR.org","journal-title":"J. Mach. Learn. Res."},{"issue":"1","key":"20_CR13","first-page":"1","volume":"22","author":"J Whitehill","year":"2009","unstructured":"Whitehill, J., Ruvolo, P., Wu, T., Bergsma, J., Movellan, J.: Whose vote should count more: optimal integration of labels from labelers of unknown expertise. Adv. Neural Inf. Process. Syst. 22(1), 1\u20139 (2009)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"20_CR14","doi-asserted-by":"crossref","unstructured":"El Maarry, K., Balke, W.-T., Cho, H., Hwang, S., Baba, Y.: Skill ontology-based model for quality assurance in crowdsourcing. In: UnCrowd 2014: DASFAA Workshop on Uncertain and Crowdsourced Data, Bali, Indonesia (2014)","DOI":"10.1007\/978-3-662-43984-5_29"},{"key":"20_CR15","doi-asserted-by":"crossref","unstructured":"Ignjatovic, A., Foo, N., Lee, C.T.L.C.T.: An analytic approach to reputation ranking of participants in online transactions. In: IEEE\/WIC\/ACM International Conferences on Web Intelligence and Intelligent Agent Technology-Workshops, vol. 1 (2008)","DOI":"10.1109\/WIIAT.2008.14"},{"issue":"2","key":"20_CR16","first-page":"97","volume":"5","author":"Z Noorian","year":"2010","unstructured":"Noorian, Z., Ulieru, M.: The state of the art in trust and reputation systems: a framework for comparison. J. Theor. Appl. Eelectron. Commer. Res. 5(2), 97\u2013117 (2010)","journal-title":"J. Theor. Appl. Eelectron. Commer. Res."},{"key":"20_CR17","doi-asserted-by":"crossref","unstructured":"Hossain, M.: Users\u2019 motivation to participate in online crowdsourcing platforms. In: ICIMTR 2012 \u2013 2012 International Conference on Innovation, Management and Technology Research, pp. 310\u2013315 (2012)","DOI":"10.1109\/ICIMTR.2012.6236409"},{"issue":"2","key":"20_CR18","first-page":"165","volume":"44","author":"G Kazai","year":"2011","unstructured":"Kazai, G.: In search of quality in crowdsourcing for search engine evaluation. SIGIR Forum 44(2), 165\u2013176 (2011)","journal-title":"SIGIR Forum"},{"issue":"6943","key":"20_CR19","doi-asserted-by":"publisher","first-page":"1552","DOI":"10.1136\/bmj.308.6943.1552","volume":"308","author":"DG Altman","year":"1994","unstructured":"Altman, D.G., Bland, J.M.: Diagnostic tests. 1: sensitivity and specificity. Br. Med. J. 308(6943), 1552 (1994). (Clinical research edition)","journal-title":"Br. Med. J."},{"issue":"7417","key":"20_CR20","doi-asserted-by":"publisher","first-page":"716","DOI":"10.1136\/bmj.327.7417.716","volume":"327","author":"T-W Loong","year":"2003","unstructured":"Loong, T.-W.: Understanding sensitivity and specificity with the right side of the brain. Br. Med. J. 327(7417), 716\u2013719 (2003). (BMJ Publishing Group Ltd.)","journal-title":"Br. Med. J."},{"issue":"6947","key":"20_CR21","doi-asserted-by":"publisher","first-page":"102","DOI":"10.1136\/bmj.309.6947.102","volume":"309","author":"DG Altman","year":"1994","unstructured":"Altman, D.G., Bland, J.M.: Diagnostic tests 2: predictive values. Br. Med. J. 309(6947), 102 (1994). (Clinical research edition)","journal-title":"Br. Med. J."},{"key":"20_CR22","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"crossref","first-page":"169","DOI":"10.1007\/978-3-319-18123-3_11","volume-title":"Database Systems for Advanced Applications","author":"K El Maarry","year":"2015","unstructured":"El Maarry, K., Balke, W.-T.: Retaining rough diamonds: towards a fairer elimination of low-skilled workers. In: Renz, M., Shahabi, C., Zhou, X., Cheema, M.A. (eds.) DASFAA 2015. LNCS, vol. 9050, pp. 169\u2013185. Springer, Heidelberg (2015)"},{"key":"20_CR23","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"17","DOI":"10.1007\/978-3-319-21042-1_2","volume-title":"Web-Age Information Management","author":"K El Maarry","year":"2015","unstructured":"El Maarry, K., G\u00fcntzer, U., Balke, W.-T.: Realizing impact sourcing by adaptive gold questions: a socially responsible measure for workers\u2019 trustworthiness. In: Dong, X.L., Yu, X., Li, J., Sun, Y. (eds.). LNCS, vol. 9098, pp. 17\u201329. Springer, Heidelberg (2015)"}],"container-title":["Lecture Notes in Computer Science","Web Information Systems Engineering \u2013 WISE 2015"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-319-26190-4_20","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,8,15]],"date-time":"2023-08-15T16:33:54Z","timestamp":1692117234000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-319-26190-4_20"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015]]},"ISBN":["9783319261898","9783319261904"],"references-count":23,"URL":"https:\/\/doi.org\/10.1007\/978-3-319-26190-4_20","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2015]]}}}