{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,30]],"date-time":"2025-03-30T10:32:40Z","timestamp":1743330760786,"version":"3.37.3"},"reference-count":40,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2018,6,29]],"date-time":"2018-06-29T00:00:00Z","timestamp":1530230400000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Int J Data Sci Anal"],"published-print":{"date-parts":[[2019,4]]},"DOI":"10.1007\/s41060-018-0139-5","type":"journal-article","created":{"date-parts":[[2018,6,29]],"date-time":"2018-06-29T09:41:30Z","timestamp":1530265290000},"page":"179-199","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Sloppiness mitigation in crowdsourcing: detecting and correcting bias for crowd scoring tasks"],"prefix":"10.1007","volume":"7","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-3770-4840","authenticated-orcid":false,"given":"Lingyu","family":"Lyu","sequence":"first","affiliation":[]},{"given":"Mehmed","family":"Kantardzic","sequence":"additional","affiliation":[]},{"given":"Tegjyot Singh","family":"Sethi","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2018,6,29]]},"reference":[{"doi-asserted-by":"crossref","unstructured":"Aydin, B.I., Yilmaz, Y.S., Li, Y., Li, Q., Gao, J., Demirbas, M. Crowdsourcing for multiple-choice question answering. In: AAAI, pp. 2946\u20132953 (2014)","key":"139_CR1","DOI":"10.1609\/aaai.v28i2.19016"},{"unstructured":"Bertsekas, D.P. Non-linear programming. In: Athena scientific (1999)","key":"139_CR2"},{"unstructured":"Buckley, C., Lease, M., Smucker, M.D., Jung, H.J., Grady, C. Overview of the TREC 2010 relevance feedback track (notebook). In: The Nineteenth Text Retrieval Conference (TREC) Notebook (2010)","key":"139_CR3"},{"key":"139_CR4","doi-asserted-by":"publisher","first-page":"20","DOI":"10.2307\/2346806","volume":"28","author":"AP Dawid","year":"1979","unstructured":"Dawid, A.P., Skene, A.M.: Maximum likelihood estimation of observer error-rates using the EM algorithm. Appl. Stat. 28, 20\u201328 (1979)","journal-title":"Appl. Stat."},{"doi-asserted-by":"crossref","unstructured":"De\u00a0Alfaro, L., Shavlovsky, M. Crowdgrader: a tool for crowdsourcing the evaluation of homework assignments. In: Proceedings of the 45th ACM Technical Symposium on Computer Science Education. ACM, pp. 415\u2013420 (2014)","key":"139_CR5","DOI":"10.1145\/2538862.2538900"},{"unstructured":"Dekel, O., Shamir, O.: Vox populi: collecting high-quality labels from a crowd (2009)","key":"139_CR6"},{"doi-asserted-by":"crossref","unstructured":"Demartini, G., Difallah, D.E., Cudr\u00e9-Mauroux, P.: Zencrowd: leveraging probabilistic reasoning and crowdsourcing techniques for large-scale entity linking. In: Proceedings of the 21st International Conference on World Wide Web. ACM, pp. 469\u2013478 (2012)","key":"139_CR7","DOI":"10.1145\/2187836.2187900"},{"issue":"1","key":"139_CR8","doi-asserted-by":"publisher","first-page":"550","DOI":"10.14778\/1687627.1687690","volume":"2","author":"XL Dong","year":"2009","unstructured":"Dong, X.L., Berti-Equille, L., Srivastava, D.: Integrating conflicting data: the role of source dependence. Proc. VLDB Endow. 2(1), 550\u2013561 (2009)","journal-title":"Proc. VLDB Endow."},{"unstructured":"Ertekin, S., Hirsh, H., Rudin, C.: Learning to predict the wisdom of crowds (2012). Preprint. arXiv:1204.3611","key":"139_CR9"},{"doi-asserted-by":"crossref","unstructured":"Galland, A., Abiteboul, S., Marian, A., Senellart, P.: Corroborating information from disagreeing views. In: Proceedings of the third ACM International Conference on Web Search and Data Mining. ACM, pp. 131\u2013140 (2010)","key":"139_CR10","DOI":"10.1145\/1718487.1718504"},{"issue":"12","key":"139_CR11","doi-asserted-by":"publisher","first-page":"2048","DOI":"10.14778\/2824032.2824136","volume":"8","author":"J Gao","year":"2015","unstructured":"Gao, J., Li, Q., Zhao, B., Fan, W., Han, J.: Truth discovery and crowdsourcing aggregation: a unified perspective. Proc. VLDB Endow. 8(12), 2048\u20132049 (2015)","journal-title":"Proc. VLDB Endow."},{"key":"139_CR12","volume-title":"Bayesian Data Analysis","author":"A Gelman","year":"2014","unstructured":"Gelman, A., Carlin, J.B., Stern, H.S., Dunson, D.B., Vehtari, A., Rubin, D.B.: Bayesian Data Analysis, vol. 2. CRC Press, Boca Raton (2014)"},{"issue":"3","key":"139_CR13","doi-asserted-by":"publisher","first-page":"791","DOI":"10.1162\/003355300554917","volume":"115","author":"U Gneezy","year":"2000","unstructured":"Gneezy, U., Rustichini, A.: Pay enough or don\u2019t pay at all. Q. J. Econ. 115(3), 791\u2013810 (2000)","journal-title":"Q. J. Econ."},{"issue":"3","key":"139_CR14","first-page":"257","volume":"50","author":"A Hama","year":"2010","unstructured":"Hama, A.: Predictably irrational: the hidden forces that shape our decisions. Mank. Q. 50(3), 257 (2010)","journal-title":"Mank. Q."},{"doi-asserted-by":"crossref","unstructured":"Ipeirotis, P.G., Gabrilovich, E.: Quizz: targeted crowdsourcing with a billion (potential) users. In: Proceedings of the 23rd International Conference on World Wide Web. ACM, pp. 143\u2013154 (2014)","key":"139_CR15","DOI":"10.1145\/2566486.2567988"},{"doi-asserted-by":"crossref","unstructured":"Ipeirotis, P.G., Provost, F., Wang, J.: Quality management on amazon mechanical turk. In: Proceedings of the ACM SIGKDD Workshop on Human Computation. ACM, pp. 64\u201367 (2010)","key":"139_CR16","DOI":"10.1145\/1837885.1837906"},{"doi-asserted-by":"crossref","unstructured":"Kamar, E., Kapoor, A., Horvitz, E.: Identifying and accounting for task-dependent bias in crowdsourcing. In: Third AAAI Conference on Human Computation and Crowdsourcing (2015)","key":"139_CR17","DOI":"10.1609\/hcomp.v3i1.13238"},{"unstructured":"Karger, D.R., Oh, S., Shah, D.: Iterative learning for reliable crowdsourcing systems. In: Advances in Neural Information Proceeding Systems, pp. 1953\u20131961 (2011)","key":"139_CR18"},{"doi-asserted-by":"crossref","unstructured":"Kazai, G., Kamps, J., Milic-Frayling, N.: Worker types and personality traits in crowdsourcing relevance labels. In: Proceedings of the 20th ACM International Conference on Information and Knowledge Management. ACM, pp. 1941\u20131944 (2011)","key":"139_CR19","DOI":"10.1145\/2063576.2063860"},{"doi-asserted-by":"crossref","unstructured":"Li, Q., Li, Y., Gao, J., Zhao, B., Fan, W., Han, J.: Resolving conflicts in heterogeneous data by truth discovery and source reliability estimation. In: Proceedings of the 2014 ACM SIGMOD International Conference on Management of Data. ACM, pp. 1187\u20131198 (2014)","key":"139_CR20","DOI":"10.1145\/2588555.2610509"},{"issue":"2","key":"139_CR21","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/2897350.2897352","volume":"17","author":"Y Li","year":"2016","unstructured":"Li, Y., Gao, J., Meng, C., Li, Q., Su, L., Zhao, B., Fan, W., Han, J.: A survey on truth discovery. ACM Sigkdd Explor. Newsl. 17(2), 1\u201316 (2016)","journal-title":"ACM Sigkdd Explor. Newsl."},{"doi-asserted-by":"crossref","unstructured":"Meng, C., Jiang, W., Li, Y., Gao, J., Su, L., Ding, H., Cheng, Y.: Truth discovery on crowd sensing of correlated entities. In: Proceedings of the 13th ACM Conference on Embedded Networked Sensor Systems. ACM, pp. 169\u2013182 (2015)","key":"139_CR22","DOI":"10.1145\/2809695.2809715"},{"key":"139_CR23","doi-asserted-by":"publisher","first-page":"311","DOI":"10.1162\/tacl_a_00185","volume":"2","author":"RJ Passonneau","year":"2014","unstructured":"Passonneau, R.J., Carpenter, B.: The benefits of a model of annotation. Trans. Assoc. Comput. Linguist. 2, 311\u2013326 (2014)","journal-title":"Trans. Assoc. Comput. Linguist."},{"unstructured":"Pasternack, J., Roth, D.: Knowing what to believe (when you already know something). In: Proceedings of the 23rd International Conference on Computational Linguistics, Association for Computational Linguistics, pp. 877\u2013885 (2010)","key":"139_CR24"},{"key":"139_CR25","first-page":"491","volume":"13","author":"VC Raykar","year":"2012","unstructured":"Raykar, V.C., Yu, S.: Eliminating spammers and ranking annotators for crowdsourced labeling tasks. J. Mach. Learn. Res. 13, 491\u2013518 (2012)","journal-title":"J. Mach. Learn. Res."},{"key":"139_CR26","first-page":"1297","volume":"11","author":"VC Raykar","year":"2010","unstructured":"Raykar, V.C., Yu, S., Zhao, L.H., Valadez, G.H., Florin, C., Bogoni, L., Moy, L.: Learning from crowds. J. Mach. Learn. Res. 11, 1297\u20131322 (2010)","journal-title":"J. Mach. Learn. Res."},{"doi-asserted-by":"crossref","unstructured":"Sheng, V.S., Provost, F., Ipeirotis, P.G.: Get another label? Improving data quality and data mining using multiple, noisy labelers. In: Proceedings of the 14th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining. ACM, pp. 614\u2013622 (2008)","key":"139_CR27","DOI":"10.1145\/1401890.1401965"},{"doi-asserted-by":"crossref","unstructured":"Snow, R., O\u2019Connor, B., Jurafsky, D., Ng, A.Y.: Cheap and fast\u2014but is it good?: Evaluating non-expert annotations for natural language tasks. In: Proceedings of the Conference on Empirical Methods in Natural Language Processing, Association for Computational Linguistics, pp. 254\u2013263 (2008)","key":"139_CR28","DOI":"10.3115\/1613715.1613751"},{"doi-asserted-by":"crossref","unstructured":"Von\u00a0Ahn, L., Dabbish, L.: Labeling images with a computer game. In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems. ACM, pp. 319\u2013326 (2004)","key":"139_CR29","DOI":"10.1145\/985692.985733"},{"issue":"5895","key":"139_CR30","doi-asserted-by":"publisher","first-page":"1465","DOI":"10.1126\/science.1160379","volume":"321","author":"L Ahn Von","year":"2008","unstructured":"Von Ahn, L., Maurer, B., McMillen, C., Abraham, D., Blum, M.: Recaptcha: human-based character recognition via web security measures. Science 321(5895), 1465\u20131468 (2008)","journal-title":"Science"},{"issue":"1","key":"139_CR31","doi-asserted-by":"publisher","first-page":"184","DOI":"10.1007\/s11263-012-0564-1","volume":"101","author":"C Vondrick","year":"2013","unstructured":"Vondrick, C., Patterson, D., Ramanan, D.: Efficiently scaling up crowdsourced video annotation. Int. J. Comput. Vis. 101(1), 184\u2013204 (2013)","journal-title":"Int. J. Comput. Vis."},{"unstructured":"Vuurens, J., de\u00a0Vries, A.P., Eickhoff, C.: How much spam can you take? An analysis of crowdsourcing results to increase accuracy. In: Proceedings of hte ACM SIGIR Workshop on Crowdsourcing for Information Retrieval (CIR11), pp. 21\u201326 (2011)","key":"139_CR32"},{"unstructured":"Wauthier, F.L., Jordan, M.I.: Bayesian bias mitigation for crowdsourcing. In: Advances in Neural Information Processing Systems, pp. 1800\u20131808 (2011)","key":"139_CR33"},{"unstructured":"Welinder, P., Branson, S., Perona, P., Belongie, S.J.: The multidimensional wisdom of crowds. In: Advances in Neural Information Processing Systems, pp. 2424\u20132432 (2010)","key":"139_CR34"},{"unstructured":"Whitehill, J., Wu, T.F., Bergsma, J., Movellan, J.R., Ruvolo, P.L.: Whose vote should count more: optimal integration of labels from labelers of unknown expertise. In: Advances in Neural Information Processing Systems, pp. 2035\u20132043 (2009)","key":"139_CR35"},{"key":"139_CR36","first-page":"1161","volume":"11","author":"Y Yan","year":"2011","unstructured":"Yan, Y., Rosales, R., Fung, G., Dy, J.G.: Active learning from crowds. ICML 11, 1161\u20131168 (2011)","journal-title":"ICML"},{"issue":"6","key":"139_CR37","doi-asserted-by":"publisher","first-page":"796","DOI":"10.1109\/TKDE.2007.190745","volume":"20","author":"X Yin","year":"2008","unstructured":"Yin, X., Han, J., Philip, S.Y.: Truth discovery with multiple conflicting information providers on the web. IEEE Trans. Knowl. Data Eng. 20(6), 796\u2013808 (2008)","journal-title":"IEEE Trans. Knowl. Data Eng."},{"unstructured":"Zaidan, O.F., Callison-Burch, C.: Crowdsourcing translation: professional quality from non-professionals. In: Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies-Volume 1, Association for Computational Linguistics, pp. 1220\u20131229 (2011)","key":"139_CR38"},{"key":"139_CR39","doi-asserted-by":"publisher","first-page":"254","DOI":"10.1016\/j.ins.2016.12.026","volume":"382","author":"J Zhang","year":"2017","unstructured":"Zhang, J., Sheng, V.S., Li, Q., Wu, J., Wu, X.: Consensus algorithms for biased labeling in crowdsourcing. Inf. Sci. 382, 254\u2013273 (2017)","journal-title":"Inf. Sci."},{"unstructured":"Zhou, D., Basu, S., Mao, Y., Platt, J.C.: Learning from the wisdom of crowds by minimax entropy. In: Advances in Neural Information Processing Systems, pp. 2195\u20132203 (2012)","key":"139_CR40"}],"container-title":["International Journal of Data Science and Analytics"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s41060-018-0139-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s41060-018-0139-5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s41060-018-0139-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,9,3]],"date-time":"2023-09-03T12:20:40Z","timestamp":1693743640000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s41060-018-0139-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,6,29]]},"references-count":40,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2019,4]]}},"alternative-id":["139"],"URL":"https:\/\/doi.org\/10.1007\/s41060-018-0139-5","relation":{},"ISSN":["2364-415X","2364-4168"],"issn-type":[{"type":"print","value":"2364-415X"},{"type":"electronic","value":"2364-4168"}],"subject":[],"published":{"date-parts":[[2018,6,29]]},"assertion":[{"value":"25 July 2017","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"8 June 2018","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"29 June 2018","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}