{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T12:34:33Z","timestamp":1776083673094,"version":"3.50.1"},"publisher-location":"Cham","reference-count":24,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783319588377","type":"print"},{"value":"9783319588384","type":"electronic"}],"license":[{"start":{"date-parts":[[2017,1,1]],"date-time":"2017-01-01T00:00:00Z","timestamp":1483228800000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2017]]},"DOI":"10.1007\/978-3-319-58838-4_7","type":"book-chapter","created":{"date-parts":[[2017,5,11]],"date-time":"2017-05-11T19:54:08Z","timestamp":1494532448000},"page":"58-68","source":"Crossref","is-referenced-by-count":3,"title":["Measuring the Quality of Annotations for a Subjective Crowdsourcing Task"],"prefix":"10.1007","author":[{"given":"Raquel","family":"Justo","sequence":"first","affiliation":[]},{"given":"M. In\u00e9s","family":"Torres","sequence":"additional","affiliation":[]},{"given":"Jos\u00e9 M.","family":"Alcaide","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2017,5,12]]},"reference":[{"issue":"4","key":"7_CR1","doi-asserted-by":"crossref","first-page":"555","DOI":"10.1162\/coli.07-034-R2","volume":"34","author":"R Artstein","year":"2008","unstructured":"Artstein, R., Poesio, M.: Inter-coder agreement for computational linguistics. Comput. Linguist. 34(4), 555\u2013596 (2008)","journal-title":"Comput. Linguist."},{"key":"7_CR2","doi-asserted-by":"crossref","first-page":"303","DOI":"10.1086\/266520","volume":"18","author":"EM Bennet","year":"1954","unstructured":"Bennet, E.M., Alpert, R., Goldstein, A.C.: Communications through limited response questioning. Public Opin. Q. 18, 303\u2013308 (1954)","journal-title":"Public Opin. Q."},{"key":"7_CR3","volume-title":"Crowdsourced Assessment of Speech Synthesis","author":"S Buchholz","year":"2013","unstructured":"Buchholz, S., Latorre, J., Yanagisawa, K.: Crowdsourced Assessment of Speech Synthesis. Wiley, Chichester (2013)"},{"issue":"4","key":"7_CR4","doi-asserted-by":"crossref","first-page":"213","DOI":"10.1037\/h0026256","volume":"70","author":"J Cohen","year":"1968","unstructured":"Cohen, J.: Weighted kappa: nominal scale agreement provision for scaled disagreement or partial credit. Psychol. Bull. 70(4), 213\u2013220 (1968)","journal-title":"Psychol. Bull."},{"issue":"1","key":"7_CR5","doi-asserted-by":"crossref","first-page":"37","DOI":"10.1177\/001316446002000104","volume":"20","author":"J Cohen","year":"1960","unstructured":"Cohen, J.: A coefficient of agreement for nominal scales. Educ. Psychol. Meas. 20(1), 37\u201346 (1960)","journal-title":"Educ. Psychol. Meas."},{"issue":"4","key":"7_CR6","doi-asserted-by":"crossref","first-page":"1047","DOI":"10.2307\/2529886","volume":"38","author":"M Davies","year":"1982","unstructured":"Davies, M., Fleiss, J.L.: Measuring agreement for multinomial data. Biometrics 38(4), 1047\u20131051 (1982)","journal-title":"Biometrics"},{"issue":"1","key":"7_CR7","doi-asserted-by":"crossref","first-page":"20","DOI":"10.2307\/2346806","volume":"28","author":"AP Dawid","year":"1979","unstructured":"Dawid, A.P., Skene, A.M.: Maximum likelihood estimation of observer error-rates using the em algorithm. Appl. Stat. 28(1), 20\u201328 (1979)","journal-title":"Appl. Stat."},{"issue":"1","key":"7_CR8","doi-asserted-by":"crossref","first-page":"71","DOI":"10.1177\/0261927X07309512","volume":"27","author":"ML Dress","year":"2008","unstructured":"Dress, M.L., Kreuz, R.J., Link, K.E., Caucci, G.M.: Regional variation in the use of sarcasm. J. Lang. Soc. Psychol. 27(1), 71\u201385 (2008)","journal-title":"J. Lang. Soc. Psychol."},{"key":"7_CR9","unstructured":"Eickhoff, C., de Vries, A.P.: How crowdsourcable is your task? In: Workshop on Crowdsourcing for Search and Data Mining (CSDM), Hong Kong, China (2011)"},{"issue":"2","key":"7_CR10","doi-asserted-by":"crossref","first-page":"121","DOI":"10.1007\/s10791-011-9181-9","volume":"16","author":"C Eickhoff","year":"2013","unstructured":"Eickhoff, C., de Vries, A.P.: Increasing cheat robustness of crowdsourcing tasks. Inf. Retrieval 16(2), 121\u2013137 (2013)","journal-title":"Inf. Retrieval"},{"key":"7_CR11","unstructured":"Filatova, E.: Irony and sarcasm: corpus generation and analysis using crowdsourcing. In: Proceedings of LREC 2012, Istanbul, Turkey, pp. 392\u2013398, 23\u201325 May 2012"},{"issue":"5","key":"7_CR12","doi-asserted-by":"crossref","first-page":"378","DOI":"10.1037\/h0031619","volume":"76","author":"J Fleiss","year":"1971","unstructured":"Fleiss, J., et al.: Measuring nominal scale agreement among many raters. Psychol. Bull. 76(5), 378\u2013382 (1971)","journal-title":"Psychol. Bull."},{"key":"7_CR13","doi-asserted-by":"crossref","unstructured":"Gadiraju, U., Kawase, R., Dietze, S., Demartini, G.: Understanding malicious behavior in crowdsourcing platforms: the case of online surveys. In: Proceedings of the ACM CHI 2015, Seoul, Republic of Korea, pp. 1631\u20131640 (2015)","DOI":"10.1145\/2702123.2702443"},{"key":"7_CR14","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"465","DOI":"10.1007\/978-3-642-14623-7_25","volume-title":"Advances in Cryptology \u2013 CRYPTO 2010","author":"R Gennaro","year":"2010","unstructured":"Gennaro, R., Gentry, C., Parno, B.: Non-interactive verifiable computing: outsourcing computation to untrusted workers. In: Rabin, T. (ed.) CRYPTO 2010. LNCS, vol. 6223, pp. 465\u2013482. Springer, Heidelberg (2010). doi: 10.1007\/978-3-642-14623-7_25"},{"key":"7_CR15","doi-asserted-by":"crossref","unstructured":"Ipeirotis, P.G., Provost, F., Wang, J.: Quality management on Amazon mechanical turk. In: Proceedings of the ACM SIGKDD, pp. 64\u201367. New York, USA (2010)","DOI":"10.1145\/1837885.1837906"},{"key":"7_CR16","unstructured":"Justo, R., Alcaide, J.M., Torres, M.I.: Crowdscience: crowdsourcing for research and development. In: Proceedings of IberSpeech 2016, Portugal, pp. 403\u2013410 (2016)"},{"key":"7_CR17","doi-asserted-by":"crossref","unstructured":"Kou, Z., Stanton, D., Peng, F., Beaufays, F., Strohman, T.: Fix it where it fails: pronunciation learning by mining error corrections from speech logs. In: Proceedings of ICASSP 2015, South Brisbane, Australia, pp. 4619\u20134623, 19\u201324 April 2015","DOI":"10.1109\/ICASSP.2015.7178846"},{"key":"7_CR18","volume-title":"Content Analysis: An Introduction to its Methodology","author":"K Krippendorff","year":"2004","unstructured":"Krippendorff, K.: Content Analysis: An Introduction to its Methodology. Sage, Thousand Oaks (2004)"},{"key":"7_CR19","doi-asserted-by":"crossref","unstructured":"Krippendorff, K.: Computing Krippendorff\u2019s Alpha Reliability. Technical report, University of Pennsylvania, Annenberg School for Communication, June 2007","DOI":"10.1002\/9781405186407.wbiecr029"},{"key":"7_CR20","volume-title":"The Way we Talk Now: Commentaries on Language and Culture from NPR\u2019s \u201cFresh Air\u201d","author":"G Nunberg","year":"2001","unstructured":"Nunberg, G.: The Way we Talk Now: Commentaries on Language and Culture from NPR\u2019s \u201cFresh Air\u201d. Houghton Mifflin, Boston (2001)"},{"issue":"12","key":"7_CR21","doi-asserted-by":"crossref","first-page":"1428","DOI":"10.1016\/j.patrec.2013.05.012","volume":"34","author":"F Rodrigues","year":"2013","unstructured":"Rodrigues, F., Pereira, F.C., Ribeiro, B.: Learning from multiple annotators: distinguishing good from random labelers. Pattern Recogn. Lett. 34(12), 1428\u20131436 (2013)","journal-title":"Pattern Recogn. Lett."},{"key":"7_CR22","doi-asserted-by":"crossref","unstructured":"Rothwell, S., Elshenawy, A., Carter, S., Iraga, D., Romani, F., Kennewick, M., Kennewick, B.: Controlling quality and handling fraud in large scale crowdsourcing speech data collections. In: Proceedings of Interspeech 2015, Dresden, Germany, pp. 2784\u20132788. ISCA, 6\u201310 September 2015","DOI":"10.21437\/Interspeech.2015-586"},{"issue":"3","key":"7_CR23","doi-asserted-by":"crossref","first-page":"321","DOI":"10.1086\/266577","volume":"19","author":"WA Scott","year":"1955","unstructured":"Scott, W.A.: Reliability of content analysis: the case of nominal scale coding. Public Opin. Q. 19(3), 321\u2013325 (1955)","journal-title":"Public Opin. Q."},{"key":"7_CR24","unstructured":"Swanson, R., Lukin, S.M., Eisenberg, L., Corcoran, T., Walker, M.A.: Getting reliable annotations for sarcasm in online dialogues. In: Proceedings of LREC 2014, Reykjavik, Iceland, pp. 4250\u20134257, 26\u201331 May 2014"}],"container-title":["Lecture Notes in Computer Science","Pattern Recognition and Image Analysis"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-319-58838-4_7","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,8,23]],"date-time":"2023-08-23T14:32:08Z","timestamp":1692801128000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-319-58838-4_7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017]]},"ISBN":["9783319588377","9783319588384"],"references-count":24,"URL":"https:\/\/doi.org\/10.1007\/978-3-319-58838-4_7","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2017]]}}}