{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,30]],"date-time":"2026-06-30T01:14:44Z","timestamp":1782782084970,"version":"3.54.5"},"reference-count":17,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2014,5,18]],"date-time":"2014-05-18T00:00:00Z","timestamp":1400371200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Soft Comput"],"published-print":{"date-parts":[[2015,6]]},"DOI":"10.1007\/s00500-014-1303-z","type":"journal-article","created":{"date-parts":[[2014,5,17]],"date-time":"2014-05-17T05:43:52Z","timestamp":1400305432000},"page":"1541-1552","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["How clumpy is my image?"],"prefix":"10.1007","volume":"19","author":[{"given":"Hugo","family":"Hutt","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Richard","family":"Everson","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Murray","family":"Grant","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"John","family":"Love","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"George","family":"Littlejohn","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2014,5,18]]},"reference":[{"key":"1303_CR1","doi-asserted-by":"crossref","DOI":"10.1017\/CBO9780511611483","volume-title":"Design of comparative experiments","author":"RA Bailey","year":"2008","unstructured":"Bailey RA (2008) Design of comparative experiments, 1st edn. Cambridge University Press, Cambridge","edition":"1"},{"issue":"4","key":"1303_CR2","doi-asserted-by":"crossref","first-page":"86","DOI":"10.1145\/1924421.1924442","volume":"54","author":"A Doan","year":"2011","unstructured":"Doan A, Ramakrishnan R, Halevy AY (2011) Crowdsourcing systems on the world-wide web. Commun ACM 54(4):86\u201396","journal-title":"Commun ACM"},{"issue":"8","key":"1303_CR3","doi-asserted-by":"crossref","first-page":"861","DOI":"10.1016\/j.patrec.2005.10.010","volume":"27","author":"T Fawcett","year":"2006","unstructured":"Fawcett T (2006) An introduction to roc analysis. Pattern Recognit Lett 27(8):861\u2013874 ISSN 0167\u20138655","journal-title":"Pattern Recognit Lett"},{"key":"1303_CR4","doi-asserted-by":"crossref","unstructured":"Fortson L, Masters K, Nichol R, Borne K, Edmondson E, Lintott C, Raddick J, Schawinski K, Wallin J (2012) Galaxy Zoo: morphological classification and citizen science. In: Way MJ, Scargle JD, Ali KM, Srivastava AN (eds) Advances in machine learning and data mining for astronomy, data mining and knowledge discovery series. Chapman and Hall\/CRC, Boca Raton","DOI":"10.1201\/b11822-16"},{"key":"1303_CR5","doi-asserted-by":"crossref","unstructured":"Gelman A, Carlin JB, Stren HS, Dunson DB, Vehtari A, Rubin DB (2013) Bayesian data analysis, 3rd edn. Chapman and Hall\/CRC, Boca Raton","DOI":"10.1201\/b16018"},{"key":"1303_CR6","doi-asserted-by":"crossref","unstructured":"Heer J, Bostock M (2010) Crowdsourcing graphical perception: using mechanical turk to assess visualization design. In: Proceedings of the SIGCHI conference on human factors in computing systems. ACM, New York, pp 203\u2013212","DOI":"10.1145\/1753326.1753357"},{"key":"1303_CR7","doi-asserted-by":"crossref","unstructured":"Knowles J, Corne D (1999) The pareto archived evolution strategy: a new baseline algorithm for pareto multiobjective optimisation. In: Proceedings of the Congress on evolutionary computation vol 1, pp 98\u2013105","DOI":"10.1109\/CEC.1999.781913"},{"key":"1303_CR8","unstructured":"Lebanon G, Lafferty J (2002) Cranking: Combining rankings using conditional probability models on permutations. In: ICML \u201902 Proceedings of the nineteenth international conference on machine learning, pp 363\u2013370"},{"key":"1303_CR9","unstructured":"Lehmann EL (2006) Nonparametrics: statistical methods based on ranks, 1st (revised) edn. Springer, New York"},{"issue":"4","key":"1303_CR10","doi-asserted-by":"crossref","first-page":"1018","DOI":"10.1111\/j.1469-8137.2010.03244.x","volume":"186","author":"GR Littlejohn","year":"2010","unstructured":"Littlejohn GR, Gouveia JD, Edner C, Smirnoff N, Love J (2010) Perfluorodecalin substantially improves confocal depth resolution in air-filled tissues. New Phytol 186(4):1018\u20131025","journal-title":"New Phytol"},{"key":"1303_CR11","unstructured":"Parent G, Eskenazi M (2010) Clustering dictionary definitions using Amazon mechanical turk. In: Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with Amazon\u2019s mechanical turk. Association for computational linguistics. Los Angeles, pp 21\u201329"},{"key":"1303_CR12","unstructured":"Raykar VC, Yu S (2012) Eliminating spammers and ranking annotators for crowdsourced labeling tasks. J Mach Learn Res 13:491\u2013518 ISSN 1532\u20134435"},{"key":"1303_CR13","first-page":"1297","volume":"11","author":"VC Raykar","year":"2010","unstructured":"Raykar VC, Yu S, Zhao LH, Valadez GH, Florin C, Bogoni L, Moy L (2010) Learning from crowds. J Mach Learn Res 11:1297\u20131322","journal-title":"J Mach Learn Res"},{"key":"1303_CR14","doi-asserted-by":"crossref","unstructured":"Snow R, O\u2019Connor B, Jurafsky D, Ng AY (2008) Cheap and fast\u2014 but is it good? Evaluating non-expert annotations for natural language tasks. In: Proceedings of the EMNLP conference on empirical methods in natural language processing. ACM, New York, pp 254\u2013263","DOI":"10.3115\/1613715.1613751"},{"key":"1303_CR15","doi-asserted-by":"crossref","unstructured":"Tibshirani R (1996) Regression shrinkage and selection via the LASSO. J Royal Stat Soc Ser B 58:267288","DOI":"10.1111\/j.2517-6161.1996.tb02080.x"},{"key":"1303_CR16","doi-asserted-by":"crossref","first-page":"14","DOI":"10.1111\/j.1365-313X.2006.02672.x","volume":"46","author":"W Truman","year":"2006","unstructured":"Truman W, de Torres Zabala M, Grant M (2006) A complex interplay of transcriptional regulation acts to modify basal defense responses during pathogenesis. Plant J 46:14\u201333","journal-title":"Plant J"},{"key":"1303_CR17","unstructured":"Whitehill J, Ruvolo P, Wu T, Bergsma J, Movellan J (2009) Whose vote should count more: optimal integration of labels from labelers of unknown expertise. In: Advances in neural information process systems vol 22, pp 2035\u20132043"}],"container-title":["Soft Computing"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s00500-014-1303-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s00500-014-1303-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s00500-014-1303-z","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,8,19]],"date-time":"2020-08-19T15:14:34Z","timestamp":1597850074000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s00500-014-1303-z"}},"subtitle":["Scoring in crowdsourced annotation tasks"],"short-title":[],"issued":{"date-parts":[[2014,5,18]]},"references-count":17,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2015,6]]}},"alternative-id":["1303"],"URL":"https:\/\/doi.org\/10.1007\/s00500-014-1303-z","relation":{},"ISSN":["1432-7643","1433-7479"],"issn-type":[{"value":"1432-7643","type":"print"},{"value":"1433-7479","type":"electronic"}],"subject":[],"published":{"date-parts":[[2014,5,18]]}}}