{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:02:36Z","timestamp":1755838956814},"reference-count":50,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2020,5,19]],"date-time":"2020-05-19T00:00:00Z","timestamp":1589846400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,5,19]],"date-time":"2020-05-19T00:00:00Z","timestamp":1589846400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["The VLDB Journal"],"published-print":{"date-parts":[[2020,11]]},"DOI":"10.1007\/s00778-020-00613-w","type":"journal-article","created":{"date-parts":[[2020,5,19]],"date-time":"2020-05-19T15:11:23Z","timestamp":1589901083000},"page":"1311-1336","update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":9,"title":["A game-based framework for crowdsourced data labeling"],"prefix":"10.1007","volume":"29","author":[{"given":"Jingru","family":"Yang","sequence":"first","affiliation":[]},{"given":"Ju","family":"Fan","sequence":"additional","affiliation":[]},{"given":"Zhewei","family":"Wei","sequence":"additional","affiliation":[]},{"given":"Guoliang","family":"Li","sequence":"additional","affiliation":[]},{"given":"Tongyu","family":"Liu","sequence":"additional","affiliation":[]},{"given":"Xiaoyong","family":"Du","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2020,5,19]]},"reference":[{"key":"613_CR1","doi-asserted-by":"crossref","unstructured":"Abad, A., Nabi, M., Moschitti, A.: Self-crowdsourcing training for relation extraction. In: ACL pp. 518\u2013523 (2017)","DOI":"10.18653\/v1\/P17-2082"},{"key":"613_CR2","volume-title":"Pattern Recognition and Machine Learning, Information Science and Statistics","author":"CM Bishop","year":"2007","unstructured":"Bishop, C.M.: Pattern Recognition and Machine Learning, Information Science and Statistics, 5th edn. Springer, Berlin (2007)","edition":"5"},{"issue":"3\u20134","key":"613_CR3","doi-asserted-by":"publisher","first-page":"217","DOI":"10.1080\/00949659208811439","volume":"43","author":"K Bowman","year":"1992","unstructured":"Bowman, K., Shenton, L.: Parameter estimation for the beta distribution. J. Stat. Comput. Simul. 43(3\u20134), 217\u2013228 (1992)","journal-title":"J. Stat. Comput. Simul."},{"key":"613_CR4","doi-asserted-by":"crossref","unstructured":"Chai, C., Li, G., Li, J., Deng, D., Feng, J.: Cost-effective crowdsourced entity resolution: a partial-order approach. In: SIGMOD, pp. 969\u2013984 (2016)","DOI":"10.1145\/2882903.2915252"},{"issue":"1","key":"613_CR5","first-page":"22","volume":"16","author":"KW Church","year":"1990","unstructured":"Church, K.W., Hanks, P.: Word association norms, mutual information, and lexicography. Comput. Linguist. 16(1), 22\u201329 (1990)","journal-title":"Comput. Linguist."},{"key":"613_CR6","doi-asserted-by":"crossref","unstructured":"Das, S., P.\u00a0S.\u00a0G. C., Doan, A., Naughton, J.\u00a0F., Krishnan, G., Deep, R., Arcaute, E., Raghavendra, V., Park, Y.: Falcon: Scaling up hands-off crowdsourced entity matching to build cloud services. In: SIGMOD, pp. 1431\u20131446 (2017)","DOI":"10.1145\/3035918.3035960"},{"issue":"1","key":"613_CR7","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/TKDE.2007.250581","volume":"19","author":"AK Elmagarmid","year":"2007","unstructured":"Elmagarmid, A.K., Ipeirotis, P.G., Verykios, V.S.: Duplicate record detection: a survey. IEEE Trans. Knowl. Data Eng. 19(1), 1\u201316 (2007)","journal-title":"IEEE Trans. Knowl. Data Eng."},{"issue":"2","key":"613_CR8","first-page":"104","volume":"41","author":"J Fan","year":"2018","unstructured":"Fan, J., Li, G.: Human-in-the-loop rule learning for data integration. IEEE Data Eng. Bull. 41(2), 104\u2013115 (2018)","journal-title":"IEEE Data Eng. Bull."},{"key":"613_CR9","doi-asserted-by":"crossref","unstructured":"Fan, J., Li, G., Ooi, B.\u00a0C., Tan, K., Feng, J.: icrowd: An adaptive crowdsourcing framework. In SIGMOD, pp. 1015\u20131030 (2015)","DOI":"10.1145\/2723372.2750550"},{"key":"613_CR10","first-page":"976","volume":"2014","author":"J Fan","year":"2014","unstructured":"Fan, J., Lu, M., Ooi, B.C., Tan, W., Zhang, M.: A hybrid machine-crowdsourcing system for matching web tables. ICDE 2014, 976\u2013987 (2014)","journal-title":"ICDE"},{"issue":"8","key":"613_CR11","doi-asserted-by":"publisher","first-page":"2078","DOI":"10.1109\/TKDE.2015.2407353","volume":"27","author":"J Fan","year":"2015","unstructured":"Fan, J., Zhang, M., Kok, S., Lu, M., Ooi, B.C.: Crowdop: Query optimization for declarative crowdsourcing systems. IEEE Trans. Knowl. Data Eng. 27(8), 2078\u20132092 (2015)","journal-title":"IEEE Trans. Knowl. Data Eng."},{"key":"613_CR12","doi-asserted-by":"crossref","unstructured":"Franklin, M.\u00a0J., Kossmann, D., Kraska, T., Ramesh, S., Xin, R.: Crowddb: answering queries with crowdsourcing. In: SIGMOD, pp. 61\u201372 (2011)","DOI":"10.1145\/1989323.1989331"},{"key":"613_CR13","doi-asserted-by":"crossref","unstructured":"Gokhale, C., Das, S., Doan, A., Naughton, J.F., Rampalli, N., Shavlik, J.W., Zhu, X.: Corleone: Hands-off crowdsourcing for entity matching. In: SIGMOD, pp. 601\u2013612 (2014)","DOI":"10.1145\/2588555.2588576"},{"key":"613_CR14","unstructured":"Goodfellow, I., Pouget-Abadie, J., Mirza, M., Xu, B., Warde-Farley, D., Ozair, S., Courville, A., Bengio, Y.: Generative adversarial nets. In: NIPS, pp. 2672\u20132680 (2014)"},{"issue":"4","key":"613_CR15","first-page":"372","volume":"9","author":"D Haas","year":"2015","unstructured":"Haas, D., Wang, J., Wu, E., Franklin, M.J.: Clamshell: Speeding up crowds for low-latency data labeling. PVLDB 9(4), 372\u2013383 (2015)","journal-title":"PVLDB"},{"key":"613_CR16","unstructured":"Hoffmann, R., Zhang, C., Ling, X., Zettlemoyer, L., Weld, D.\u00a0S.: Knowledge-based weak supervision for information extraction of overlapping relations. In: Association for Computational Linguistics ACL, pp. 541\u2013550 (2011)"},{"key":"613_CR17","doi-asserted-by":"publisher","unstructured":"Joglekar, M., Garcia-Molina, H., Parameswaran, A.: Comprehensive and reliable crowd assessment algorithms. In: Gehrke, J., Lehner, W., Shim, K., Cha, S.K., Lohman, G.M. (eds) ICDE. IEEE Computer Society, pp. 195\u2013206. (2015) https:\/\/doi.org\/10.1109\/ICDE.2015.7113284","DOI":"10.1109\/ICDE.2015.7113284"},{"key":"613_CR18","doi-asserted-by":"crossref","unstructured":"Khan, A.R., Garcia-Molina, H.: Attribute-based crowd entity resolution. In: CIKM, pp. 549\u2013558 (2016)","DOI":"10.1145\/2983323.2983831"},{"key":"613_CR19","first-page":"957","volume":"2015","author":"MJ Kusner","year":"2015","unstructured":"Kusner, M.J., Sun, Y., Kolkin, N.I., Weinberger, K.Q.: From word embeddings to document distances. ICML 2015, 957\u2013966 (2015)","journal-title":"ICML"},{"issue":"7553","key":"613_CR20","doi-asserted-by":"publisher","first-page":"436","DOI":"10.1038\/nature14539","volume":"521","author":"Y LeCun","year":"2015","unstructured":"LeCun, Y., Bengio, Y., Hinton, G.E.: Deep learning. Nature 521(7553), 436\u2013444 (2015)","journal-title":"Nature"},{"issue":"12","key":"613_CR21","first-page":"2006","volume":"10","author":"G Li","year":"2017","unstructured":"Li, G.: Human-in-the-loop data integration. PVLDB 10(12), 2006\u20132017 (2017)","journal-title":"PVLDB"},{"key":"613_CR22","doi-asserted-by":"crossref","unstructured":"Li, G., Chai, C., Fan, J., Weng, X., Li, J., Zheng, Y., Li, Y., Yu, X., Zhang, X., Yuan, H.: CDB: optimizing queries with crowd-based selections and joins. In: SIGMOD, pp. 1463\u20131478 (2017)","DOI":"10.1145\/3035918.3064036"},{"issue":"9","key":"613_CR23","doi-asserted-by":"publisher","first-page":"2296","DOI":"10.1109\/TKDE.2016.2535242","volume":"28","author":"G Li","year":"2016","unstructured":"Li, G., Wang, J., Zheng, Y., Franklin, M.J.: Crowdsourced data management: a survey. IEEE Trans. Knowl. Data Eng. 28(9), 2296\u20132319 (2016)","journal-title":"IEEE Trans. Knowl. Data Eng."},{"key":"613_CR24","doi-asserted-by":"crossref","unstructured":"Liu, A., Soderland, S., Bragg, J., Lin, C.H., Ling, X., Weld, D.S.: Effective crowd annotation for relation extraction. In: NAACL HLT, pp. 897\u2013906 (2016)","DOI":"10.18653\/v1\/N16-1104"},{"issue":"10","key":"613_CR25","first-page":"1040","volume":"5","author":"X Liu","year":"2012","unstructured":"Liu, X., Lu, M., Ooi, B.C., Shen, Y., Wu, S., Zhang, M.: CDAS: a crowdsourcing data analytics system. PVLDB 5(10), 1040\u20131051 (2012)","journal-title":"PVLDB"},{"key":"613_CR26","first-page":"1315","volume":"2011","author":"A Marcus","year":"2011","unstructured":"Marcus, A., Wu, E., Karger, D.R., Madden, S., Miller, R.C.: Demonstration of qurk: a query processor for humanoperators. SIGMOD 2011, 1315\u20131318 (2011)","journal-title":"SIGMOD"},{"key":"613_CR27","unstructured":"Mikolov, T., Chen, K., Corrado, G., Dean, J.: Efficient estimation of word representations in vector space. arXiv preprint arXiv:1301.3781, (2013)"},{"key":"613_CR28","unstructured":"Mikolov, T., Sutskever, I., Chen, K., Corrado, G.S., Dean, J.: Distributed representations of words and phrases and their compositionality. In: NIPS, pp. 3111\u20133119 (2013)"},{"key":"613_CR29","first-page":"1003","volume":"2009","author":"M Mintz","year":"2009","unstructured":"Mintz, M., Bills, S., Snow, R., Jurafsky, D.: Distant supervision for relation extraction without labeled data. ACL 2009, 1003\u20131011 (2009)","journal-title":"ACL"},{"issue":"4","key":"613_CR30","doi-asserted-by":"publisher","first-page":"1253","DOI":"10.1073\/pnas.1219097111","volume":"111","author":"F Parisi","year":"2014","unstructured":"Parisi, F., Strino, F., Nadler, B., Kluger, Y.: Ranking and combining multiple predictors without labeled data. Proc. Natl. Acad. Sci. USA 111(4), 1253\u20138 (2014)","journal-title":"Proc. Natl. Acad. Sci. USA"},{"issue":"12","key":"613_CR31","first-page":"1990","volume":"5","author":"H Park","year":"2012","unstructured":"Park, H., Pang, R., Parameswaran, A.G., Garcia-Molina, H., Polyzotis, N., Widom, J.: Deco: a system for declarative crowdsourcing. PVLDB 5(12), 1990\u20131993 (2012)","journal-title":"PVLDB"},{"issue":"3","key":"613_CR32","first-page":"269","volume":"11","author":"A Ratner","year":"2017","unstructured":"Ratner, A., Bach, S.H., Ehrenberg, H.R., Fries, J.A., Wu, S., R\u00e9, C.: Snorkel: rapid training data creation with weak supervision. PVLDB 11(3), 269\u2013282 (2017)","journal-title":"PVLDB"},{"key":"613_CR33","first-page":"3567","volume":"2016","author":"AJ Ratner","year":"2016","unstructured":"Ratner, A.J., Sa, C.D., Wu, S., Selsam, D., R\u00e9, C.: Data programming: creating large training sets, quickly. NIPS 2016, 3567\u20133575 (2016)","journal-title":"NIPS"},{"key":"613_CR34","unstructured":"Roth, B., Klakow, D.: Combining generative and discriminative model scores for distant supervision. In: EMNLP, pp. 24\u201329 (2013)"},{"key":"613_CR35","doi-asserted-by":"publisher","unstructured":"Rubner, Y., Tomasi, C., Guibas, L.J.: A metric for distributions with applications to image databases. ICCV, IEEE Computer Society, pp. 59\u201366 (1998). https:\/\/doi.org\/10.1109\/ICCV.1998.710701","DOI":"10.1109\/ICCV.1998.710701"},{"key":"613_CR36","doi-asserted-by":"crossref","unstructured":"Sheng, V.S., Provost, F., Ipeirotis, P.G.: Get another label? improving data quality and data mining using multiple, noisy labelers. In: SIGKDD, pp. 614\u2013622. ACM (2008)","DOI":"10.1145\/1401890.1401965"},{"key":"613_CR37","doi-asserted-by":"crossref","unstructured":"Sun, C., Shrivastava, A., Singh, S., Gupta, A.: Revisiting unreasonable effectiveness of data in deep learning era. CoRR, arXiv:1707.02968 (2017)","DOI":"10.1109\/ICCV.2017.97"},{"key":"613_CR38","unstructured":"Takamatsu, S., Sato, I., Nakagawa, H.: Reducing wrong labels in distant supervision for relation extraction. In: Meeting of the Association for Computational Linguistics: Long Papers, pp. 721\u2013729 (2012)"},{"issue":"8","key":"613_CR39","doi-asserted-by":"publisher","first-page":"1588","DOI":"10.1109\/TKDE.2018.2797962","volume":"30","author":"Y Tong","year":"2018","unstructured":"Tong, Y., Chen, L., Zhou, Z., Jagadish, H.V., Shou, L., Lv, W.: Slade: a smart large-scale task decomposer in crowdsourcing. IEEE Trans. Knowl. Data Eng. 30(8), 1588\u20131601 (2018)","journal-title":"IEEE Trans. Knowl. Data Eng."},{"key":"613_CR40","doi-asserted-by":"crossref","unstructured":"Tong, Y., She, J., Ding, B., Wang, L., Chen, L.: Online mobile micro-task allocation in spatial crowdsourcing. In: ICDE, pp. 49\u201360 (2016)","DOI":"10.1109\/ICDE.2016.7498228"},{"key":"613_CR41","doi-asserted-by":"crossref","unstructured":"Verroios, V., Garcia-Molina, H., Papakonstantinou, Y.: Waldo: An adaptive human interface for crowd entity resolution. In: SIGMOD, pp. 1133\u20131148 (2017)","DOI":"10.1145\/3035918.3035931"},{"key":"613_CR42","doi-asserted-by":"crossref","unstructured":"Vesdapunt, N., Bellare, K., Dalvi, N.N.: Crowdsourcing algorithms for entity resolution. In: PVLDB (2014)","DOI":"10.14778\/2732977.2732982"},{"key":"613_CR43","doi-asserted-by":"crossref","unstructured":"Wang, J., Kraska, T., Franklin, M.J., Feng, J.: Crowder: Crowdsourcing entity resolution. In: PVLDB (2012)","DOI":"10.14778\/2350229.2350263"},{"key":"613_CR44","doi-asserted-by":"crossref","unstructured":"Wang, J., Li, G., Kraska, T., Franklin, M.J., Feng, J.: Leveraging transitive relations for crowdsourced joins. In: SIGMOD, pp. 229\u2013240 (2013)","DOI":"10.1145\/2463676.2465280"},{"key":"613_CR45","doi-asserted-by":"crossref","unstructured":"Wang, J., Yu, L., Zhang, W., Gong, Y., Xu, Y., Wang, B., Zhang, P., Zhang, D.: Irgan: a minimax game for unifying generative and discriminative information retrieval models. In: SIGIR, pp. 515\u2013524. ACM (2017)","DOI":"10.1145\/3077136.3080786"},{"key":"613_CR46","doi-asserted-by":"crossref","unstructured":"Wang, S., Xiao, X., Lee, C.: Crowd-based deduplication: an adaptive approach. In: SIGMOD, pp. 1263\u20131277 (2015)","DOI":"10.1145\/2723372.2723739"},{"issue":"6","key":"613_CR47","first-page":"349","volume":"6","author":"SE Whang","year":"2013","unstructured":"Whang, S.E., Lofgren, P., Garcia-Molina, H.: Question selection for crowd entity resolution. PVLDB 6(6), 349\u2013360 (2013)","journal-title":"PVLDB"},{"key":"613_CR48","unstructured":"Zhang, Y., Chen, X., Zhou, D., Jordan, M.I.: Spectral methods meet EM: a provably optimal algorithm for crowdsourcing. In: International Conference on Neural Information Processing Systems, pp. 1260\u20131268 (2014)"},{"issue":"5","key":"613_CR49","first-page":"541","volume":"10","author":"Y Zheng","year":"2017","unstructured":"Zheng, Y., Li, G., Li, Y., Shan, C., Cheng, R.: Truth inference in crowdsourcing: is the problem solved? PVLDB 10(5), 541\u2013552 (2017)","journal-title":"PVLDB"},{"key":"613_CR50","doi-asserted-by":"crossref","unstructured":"Zheng, Y., Wang, J., Li, G., Cheng, R., Feng, J.: QASCA: a quality-aware task assignment system for crowdsourcing applications. In: SIGMOD, pp. 1031\u20131046 (2015)","DOI":"10.1145\/2723372.2749430"}],"container-title":["The VLDB Journal"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00778-020-00613-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00778-020-00613-w\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00778-020-00613-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,5,18]],"date-time":"2021-05-18T23:50:53Z","timestamp":1621381853000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00778-020-00613-w"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,5,19]]},"references-count":50,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2020,11]]}},"alternative-id":["613"],"URL":"https:\/\/doi.org\/10.1007\/s00778-020-00613-w","relation":{},"ISSN":["1066-8888","0949-877X"],"issn-type":[{"value":"1066-8888","type":"print"},{"value":"0949-877X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,5,19]]},"assertion":[{"value":"16 September 2019","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 January 2020","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"12 April 2020","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 May 2020","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}