{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,25]],"date-time":"2026-02-25T21:19:24Z","timestamp":1772054364098,"version":"3.50.1"},"reference-count":84,"publisher":"Springer Science and Business Media LLC","issue":"31","license":[{"start":{"date-parts":[[2022,1,17]],"date-time":"2022-01-17T00:00:00Z","timestamp":1642377600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,1,17]],"date-time":"2022-01-17T00:00:00Z","timestamp":1642377600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61976187"],"award-info":[{"award-number":["61976187"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["92046002"],"award-info":[{"award-number":["92046002"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Neural Comput & Applic"],"published-print":{"date-parts":[[2023,11]]},"DOI":"10.1007\/s00521-021-06825-7","type":"journal-article","created":{"date-parts":[[2022,1,17]],"date-time":"2022-01-17T00:03:47Z","timestamp":1642377827000},"page":"22911-22933","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["Hint: harnessing the wisdom of crowds for handling multi-phase tasks"],"prefix":"10.1007","volume":"35","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4599-5761","authenticated-orcid":false,"given":"Yili","family":"Fang","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Pengpeng","family":"Chen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Tao","family":"han","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2022,1,17]]},"reference":[{"issue":"6","key":"6825_CR1","first-page":"1","volume":"14","author":"J Howe","year":"2006","unstructured":"Howe J (2006) The rise of crowdsourcing. Wired Mag 14(6):1\u20134","journal-title":"Wired Mag"},{"key":"6825_CR2","unstructured":"Kamar E, Hacker S, Horvitz E (2012) Combining human and machine intelligence in large-scale crowdsourcing. In: Proceedings of the 11th International Conference on Autonomous Agents and Multiagent Systems, AAMAS\u20192012. International Foundation for Autonomous Agents and Multiagent Systems, pp. 467\u2013474"},{"key":"6825_CR3","doi-asserted-by":"publisher","unstructured":"Lampinen A, Lutz C, Newlands G, Light A, Immorlica N (2018) Power struggles in the digital economy: platforms, workers, and markets. In: Companion of the 2018 ACM Conference on Computer Supported Cooperative Work and Social Computing, CSCW\u201918. Association for Computing Machinery, New York, NY, USA, pp. 417\u2013423 https:\/\/doi.org\/10.1145\/3272973.3273004","DOI":"10.1145\/3272973.3273004"},{"key":"6825_CR4","doi-asserted-by":"crossref","unstructured":"Quinn AJ, Bederson BB (2011) Human computation: a survey and taxonomy of a growing field. In: Proceedings of the International Conference on Human Factors in Computing Systems, CHI\u20192011, pp. 1403\u20131412","DOI":"10.1145\/1978942.1979148"},{"key":"6825_CR5","unstructured":"Han T, Sun H, Song Y, Fang Y, Liu X (2016) Incorporating external knowledge into crowd intelligence for more specific knowledge acquisition. In: Proceedings of the Twenty-Fifth International Joint Conference on Artificial Intelligence, IJCAI\u20192016, AAAI Press, pp. 1541\u20131547"},{"issue":"5895","key":"6825_CR6","doi-asserted-by":"publisher","first-page":"1465","DOI":"10.1126\/science.1160379","volume":"321","author":"L Von Ahn","year":"2008","unstructured":"Von Ahn L, Maurer B, McMillen C, Abraham D, Blum M (2008) reCAPTCHA: human-based character recognition via web security measures. Science 321(5895):1465\u20131468","journal-title":"Science"},{"key":"6825_CR7","doi-asserted-by":"crossref","unstructured":"Teevan J, Iqbal ST, von Veh C (2016) Supporting collaborative writing with microtasks. In: Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems, CHI\u20192016, ACM, pp. 2657\u20132668","DOI":"10.1145\/2858036.2858108"},{"key":"6825_CR8","doi-asserted-by":"crossref","unstructured":"Stol KJ, Fitzgerald B (2014) Two\u2019s company, three\u2019s a crowd: a case study of crowdsourcing software development. In: Proceedings of the 36th International Conference on Software Engineering, ICSE\u20192012, ACM, pp. 187\u2013198","DOI":"10.1145\/2568225.2568249"},{"issue":"4","key":"6825_CR9","doi-asserted-by":"publisher","first-page":"295","DOI":"10.1080\/10447318.2018.1427828","volume":"34","author":"TJ Anderson","year":"2018","unstructured":"Anderson TJ, Soden R, Keegan B, Palen L, Anderson KM (2018) The crowd is the territory: assessing quality in peer-produced spatial data during disasters. Int J Hum Comput Interact. 34(4):295\u2013310","journal-title":"Int J Hum Comput Interact."},{"issue":"12","key":"6825_CR10","doi-asserted-by":"publisher","first-page":"2048","DOI":"10.14778\/2824032.2824136","volume":"8","author":"J Gao","year":"2015","unstructured":"Gao J, Li Q, Zhao B, Fan W, Han J (2015) Truth discovery and crowdsourcing aggregation: a unified perspective. Proc VLDB Endow 8(12):2048\u20132049","journal-title":"Proc VLDB Endow"},{"issue":"9","key":"6825_CR11","doi-asserted-by":"publisher","first-page":"2296","DOI":"10.1109\/TKDE.2016.2535242","volume":"28","author":"G Li","year":"2016","unstructured":"Li G, Wang J, Zheng Y, Franklin MJ (2016) Crowdsourced data management: a survey. IEEE Trans Knowl Data Eng 28(9):2296\u20132319","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"6825_CR12","unstructured":"Zhou D, Liu Q, Platt JC, Meek C (2014) Aggregating ordinal labels from crowds by minimax conditional entropy. In: Proceedings of the 31th International Conference on Machine Learning, ICML\u20192014, pp. 262\u2013270"},{"key":"6825_CR13","doi-asserted-by":"crossref","unstructured":"Sheng VS, Provost F, Ipeirotis PG (2008) Get another label? improving data quality and data mining using multiple, noisy labelers. In: KDD2008, KDD\u20192008, ACM, pp. 614\u2013622","DOI":"10.1145\/1401890.1401965"},{"key":"6825_CR14","unstructured":"Li H (2015) Theoretical analysis and efficient algorithms for crowdsourcing. Ph.D. thesis, University of California, Berkeley, USA. http:\/\/www.escholarship.org\/uc\/item\/9rd5255z"},{"key":"6825_CR15","doi-asserted-by":"crossref","unstructured":"Li H, Liu Q (2015) Cheaper and better: selecting good workers for crowdsourcing. In: Proceedings of the Third AAAI Conference on Human Computation and Crowdsourcing, HCOMP 2015, November 8\u201311, 2015, San Diego, California, USA, pp. 20\u201321","DOI":"10.1609\/hcomp.v3i1.13248"},{"key":"6825_CR16","doi-asserted-by":"crossref","unstructured":"Sheshadri A, Lease M (2013) Square: a benchmark for research on computing crowd consensus. In: Hartman B, Horvitz E (eds.) HCOMP. AAAI","DOI":"10.1609\/hcomp.v1i1.13088"},{"key":"6825_CR17","unstructured":"Gaunt A, Borsa D, Bachrach Y (2016) Training neural nets to aggregate crowdsourced responses. In: Proceedings of the Thirty-Second Conference on Uncertainty in Artificial Intelligence, UAI 2016, June 25\u201329, 2016, New York City, NY, USA. AUAI Press"},{"key":"6825_CR18","first-page":"1","volume-title":"Macrotask Crowdsourcing - engaging the crowds to address complex problems, human-computer interaction series","author":"I Lykourentzou","year":"2019","unstructured":"Lykourentzou I, Khan V, Papangelis K, Markopoulos P (2019) Macrotask crowdsourcing: an integrated definition. In: Khan V, Papangelis K, Lykourentzou I, Markopoulos P (eds) Macrotask Crowdsourcing - engaging the crowds to address complex problems, human-computer interaction series. Springer, New York, pp 1\u201313"},{"issue":"8","key":"6825_CR19","doi-asserted-by":"publisher","first-page":"692","DOI":"10.1080\/10447318.2018.1550179","volume":"35","author":"T Roper","year":"2019","unstructured":"Roper T, Dutka LM, Cobb S, Patel H (2019) Collaborative virtual environment to facilitate game design evaluation with children with ASC. Int J Hum Comput Interact 35(8):692\u2013705","journal-title":"Int J Hum Comput Interact"},{"issue":"4","key":"6825_CR20","first-page":"14","volume":"39","author":"L Chen","year":"2016","unstructured":"Chen L, Shahabi C (2016) Spatial crowdsourcing: challenges and opportunities. IEEE Data Eng Bull 39(4):14\u201325","journal-title":"IEEE Data Eng Bull"},{"key":"6825_CR21","doi-asserted-by":"crossref","unstructured":"Dow S, Kulkarni A, Bunge B, Nguyen T, Klemmer S, Hartmann B (2011) Shepherding the crowd: managing and providing feedback to crowd workers. In: CHI\u201911 Extended Abstracts on Human Factors in Computing Systems, CHI\u20192011, ACM, pp. 1669\u20131674","DOI":"10.1145\/1979742.1979826"},{"key":"6825_CR22","doi-asserted-by":"crossref","unstructured":"Fang Y, Sun H, Li G, Zhang R, Huai J (2016) Effective result inference for context-sensitive tasks in crowdsourcing. In: DASFAA2016, DASFAA\u20192016, pp. 33\u201348","DOI":"10.1007\/978-3-319-32025-0_3"},{"key":"6825_CR23","doi-asserted-by":"crossref","unstructured":"Ryskeldiev B, Igarashi T, Zhang J, Ochiai Y, Cohen M, Herder J (2018) Spotility: crowdsourced telepresence for social and collaborative experiences in mobile mixed reality. In: Companion of the 2018 ACM Conference on Computer Supported Cooperative Work and Social Computing, CSCW 2018, Jersey City, NJ, USA, November 03\u201307, 2018, pp. 373\u2013376","DOI":"10.1145\/3272973.3274100"},{"key":"6825_CR24","doi-asserted-by":"publisher","first-page":"50","DOI":"10.1016\/j.ijhcs.2017.03.003","volume":"104","author":"J Sprinks","year":"2017","unstructured":"Sprinks J, Wardlaw J, Houghton R, Bamford S, Morley J (2017) Task workflow design and its impact on performance and volunteers\u2019 subjective preference in virtual citizen science. Int J Hum Comput Stud 104:50\u201363","journal-title":"Int J Hum Comput Stud"},{"key":"6825_CR25","doi-asserted-by":"crossref","unstructured":"Zhu Y, Yue S, Yu C, Shi Y (2017) Cept: collaborative editing tool for non-native authors. In: Proceedings of the 2017 ACM Conference on Computer Supported Cooperative Work and Social Computing, CSCW\u20192017, ACM, pp. 273\u2013285","DOI":"10.1145\/2998181.2998306"},{"issue":"11","key":"6825_CR26","doi-asserted-by":"publisher","first-page":"1106","DOI":"10.1109\/TSE.2018.2823327","volume":"45","author":"TD LaToza","year":"2019","unstructured":"LaToza TD, Lecce AD, Ricci F, Towne WB, van der Hoek A (2019) Microtask programming. IEEE Trans Softw Eng 45(11):1106\u20131124","journal-title":"IEEE Trans Softw Eng"},{"key":"6825_CR27","doi-asserted-by":"crossref","unstructured":"Chilton LB, Little G, Edge D, Weld DS, Landay JA (2013) Cascade: crowdsourcing taxonomy creation. In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems, CHI\u20192013, ACM, pp. 1999\u20132008","DOI":"10.1145\/2470654.2466265"},{"key":"6825_CR28","doi-asserted-by":"crossref","unstructured":"Lotosh I, Milo T, Novgorodov S (2013) Crowdplanr: planning made easy with crowd. In: ICDE 2013, pp. 1344\u20131347","DOI":"10.1109\/ICDE.2013.6544940"},{"key":"6825_CR29","doi-asserted-by":"crossref","unstructured":"Tian S, Zhang AX, Karger D (2020) A system for interleaving discussion and summarization in collaborative document writing. In: Conference Companion Publication of the 2020 on Computer Supported Cooperative Work and Social Computing, Association for Computing Machinery, New York, NY, USA, pp. 590\u201363","DOI":"10.1145\/3406865.3418569"},{"issue":"1","key":"6825_CR30","doi-asserted-by":"publisher","first-page":"33","DOI":"10.1075\/ts.1.03mes","volume":"1","author":"M Mesipuu","year":"2012","unstructured":"Mesipuu M (2012) Translation crowdsourcing and user-translator motivation at facebook and skype. Transl Spaces 1(1):33\u201353","journal-title":"Transl Spaces"},{"key":"6825_CR31","unstructured":"Long TT, Huynh TD, Rosenfeld A, Ramchurn SD, Jennings NR (2015) Crowdsourcing complex workflows under budget constraints. In: Proceedings of the Twenty-Ninth AAAI Conference on Artificial Intelligence, AAAI\u20192015, AAAI Press, pp. 1298\u20131304"},{"key":"6825_CR32","first-page":"17","volume-title":"Crowdsourcing coordination: a review and research agenda for crowdsourcing coordination used for macro-tasks","author":"S Kim","year":"2019","unstructured":"Kim S, Robert LP Jr (2019) Crowdsourcing coordination: a review and research agenda for crowdsourcing coordination used for macro-tasks. Springer International Publishing, Cham, pp 17\u201343"},{"key":"6825_CR33","volume-title":"Macrotask crowdsourcing - engaging the crowds to address complex problems. Human-computer interaction series","year":"2019","unstructured":"Khan V, Papangelis K, Lykourentzou I, Markopoulos P (eds) (2019) Macrotask crowdsourcing - engaging the crowds to address complex problems. Human-computer interaction series. Springer, New York"},{"key":"6825_CR34","doi-asserted-by":"publisher","first-page":"585","DOI":"10.1007\/978-3-319-07293-7_57","volume-title":"HCI in Business","author":"P Brandtner","year":"2014","unstructured":"Brandtner P, Auinger A, Helfert M (2014) Principles of human computer interaction in crowdsourcing to foster motivation in the context of open innovation. In: Nah FFH (ed) HCI in Business. Springer International Publishing, Cham, pp 585\u2013596"},{"key":"6825_CR35","doi-asserted-by":"crossref","unstructured":"Fang Y, Sun H, Zhang R, Huai J, Mao Y (2014) A model for aggregating contributions of synergistic crowdsourcing workflows. In: Proceedings of the Twenty-Eighth AAAI Conference on Artificial Intelligence, pp. 3102\u20133103","DOI":"10.1609\/aaai.v28i1.9091"},{"issue":"9","key":"6825_CR36","doi-asserted-by":"publisher","first-page":"2246","DOI":"10.1109\/TKDE.2016.2555805","volume":"28","author":"C Inasu","year":"2016","unstructured":"Inasu C, Amer-YahiaSihem Chen-lei (2016) A survey of general-purpose crowdsourcing techniques. IEEE Trans Knowl Data Eng 28(9):2246\u20132266","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"6825_CR37","doi-asserted-by":"publisher","first-page":"346","DOI":"10.1016\/j.ins.2018.05.050","volume":"460\u2013461","author":"Y Fang","year":"2018","unstructured":"Fang Y, Sun H, Li G, Zhang R, Huai J (2018) Context-aware result inference in crowdsourcing. Inf Sci 460\u2013461:346\u2013363","journal-title":"Inf Sci"},{"key":"6825_CR38","doi-asserted-by":"crossref","unstructured":"Huang G, Wu MH, Quinn AJ (2021) Task design for crowdsourcing complex cognitive skills. In: Extended Abstracts of the 2021 CHI Conference on Human Factors in Computing Systems, pp. 1\u20137","DOI":"10.1145\/3411763.3443447"},{"issue":"4","key":"6825_CR39","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s11704-020-9364-x","volume":"15","author":"T Han","year":"2021","unstructured":"Han T, Sun H, Song Y, Fang Y, Liu X (2021) Find truth in the hands of the few: acquiring specific knowledge with crowdsourcing. Front Comput Sci 15(4):1\u201312","journal-title":"Front Comput Sci"},{"key":"6825_CR40","doi-asserted-by":"crossref","unstructured":"Yin B, Zeng W, Wei X (2021) Cost-effective crowdsourced join queries for entity resolution without prior knowledge. Future Gener Comput Syst","DOI":"10.1016\/j.future.2021.09.008"},{"issue":"CSCW2","key":"6825_CR41","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3479503","volume":"5","author":"S Asthana","year":"2021","unstructured":"Asthana S, Tobar Thommel S, Halfaker AL, Banovic N (2021) Automatically labeling low quality content on wikipedia by leveraging patterns in editing behaviors. Proc ACM Hum Comput Interact 5(CSCW2):1\u201323","journal-title":"Proc ACM Hum Comput Interact"},{"key":"6825_CR42","first-page":"1297","volume":"11","author":"VC Raykar","year":"2010","unstructured":"Raykar VC, Yu S, Zhao LH, Valadez GH, Florin C, Bogoni L, Moy L (2010) Learning from crowds. Mach Learn 11:1297\u20131322","journal-title":"Mach Learn"},{"key":"6825_CR43","unstructured":"Liu Q, Peng J, Ihler AT (2012) Variational inference for crowdsourcing. In: Proceedings of the 26th Annual Conference on Neural Information Processing Systems, NIPS\u20192016, pp. 701\u2013709"},{"key":"6825_CR44","doi-asserted-by":"publisher","first-page":"20","DOI":"10.2307\/2346806","volume":"28","author":"AP Dawid","year":"1979","unstructured":"Dawid AP, Skene AM (1979) Maximum likelihood estimation of observer error-rates using the em algorithm. Appl Stat 28:20\u201328","journal-title":"Appl Stat"},{"key":"6825_CR45","doi-asserted-by":"crossref","unstructured":"Fang Y, Sun H, Chen P, Huai J (2018) On the cost complexity of crowdsourcing. In: Proceedings of the Twenty-Seventh International Joint Conference on Artificial Intelligence, IJCAI\u20192018, pp. 1531\u20131537","DOI":"10.24963\/ijcai.2018\/212"},{"key":"6825_CR46","unstructured":"Whitehill J, fan Wu T, Bergsma J, Movellan JR, Ruvolo PL (2009) Whose vote should count more: Optimal integration of labels from labelers of unknown expertise. In: Y.\u00a0Bengio, D.\u00a0Schuurmans, J.D. Lafferty, C.K.I. Williams, A.\u00a0Culotta (eds.) Advances in Neural Information Processing Systems 22, pp. 2035\u20132043"},{"issue":"1","key":"6825_CR47","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10458-012-9200-2","volume":"27","author":"G Shani","year":"2013","unstructured":"Shani G, Pineau J, Kaplow R (2013) A survey of point-based pomdp solvers. Auton Agents Multi-Agent Syst 27(1):1\u201351","journal-title":"Auton Agents Multi-Agent Syst"},{"key":"6825_CR48","doi-asserted-by":"crossref","unstructured":"Zhao Q, Tong L, Swami A, Chen Y (2007) Decentralized cognitive mac for opportunistic spectrum access in ad hoc networks: a pomdp framework. California Univ Davis Dept of Electrical and Computer engineering, Tech. rep","DOI":"10.1109\/JSAC.2007.070409"},{"key":"6825_CR49","doi-asserted-by":"crossref","unstructured":"Yu L, Nickerson JV (2011) Cooks or cobblers? crowd creativity through combination. In: Proceedings of the SIGCHI conference on human factors in computing systems, pp. 1393\u20131402","DOI":"10.1145\/1978942.1979147"},{"issue":"5","key":"6825_CR50","doi-asserted-by":"publisher","first-page":"424","DOI":"10.1109\/THMS.2020.2984743","volume":"50","author":"A Rapp","year":"2020","unstructured":"Rapp A, Cena F, Schifanella C, Boella G (2020) Finding a secure place: a map-based crowdsourcing system for people with autism. IEEE Trans Hum Mach Syst 50(5):424\u2013433","journal-title":"IEEE Trans Hum Mach Syst"},{"issue":"1","key":"6825_CR51","doi-asserted-by":"publisher","first-page":"217","DOI":"10.1007\/s00778-019-00568-7","volume":"29","author":"Y Tong","year":"2020","unstructured":"Tong Y, Zhou Z, Zeng Y, Chen L, Shahabi C (2020) Spatial crowdsourcing: a survey. VLDB J 29(1):217\u2013250","journal-title":"VLDB J"},{"key":"6825_CR52","doi-asserted-by":"crossref","unstructured":"de\u00a0Boer V, Hildebrand M, Aroyo L, De\u00a0Leenheer P, Dijkshoorn C, Tesfa B, Schreiber G (2012) Nichesourcing: Harnessing the power of crowds of experts. In: Proceedings of the 18th International Conference on Knowledge Engineering and Knowledge Management, EKAW\u20192012, pp. 16\u201320","DOI":"10.1007\/978-3-642-33876-2_3"},{"key":"6825_CR53","doi-asserted-by":"crossref","unstructured":"Hosseini M, Phalp K, Taylor J, Ali R (2014) The four pillars of crowdsourcing: a reference model. In: IEEE Eighth International Conference on Research Challenges in Information Science, RCIS\u20192014, pp. 1\u201312. IEEE","DOI":"10.1109\/RCIS.2014.6861072"},{"key":"6825_CR54","unstructured":"Hetmank L (2013) Components and functions of crowdsourcing systems-a systematic literature review. Wirtschaftsinformatik"},{"issue":"5","key":"6825_CR55","doi-asserted-by":"publisher","first-page":"13","DOI":"10.1109\/MIC.2012.68","volume":"16","author":"RR Morris","year":"2012","unstructured":"Morris RR, Dontcheva M, Gerber EM (2012) Priming for better performance in microtask crowdsourcing environments. IEEE Internet Comput 16(5):13\u201319","journal-title":"IEEE Internet Comput"},{"key":"6825_CR56","doi-asserted-by":"publisher","first-page":"794","DOI":"10.1109\/TKDE.2014.2327026","volume":"27","author":"A Kurve","year":"2015","unstructured":"Kurve A, Miller DJ, Kesidis G (2015) Multicategory crowdsourcing accounting for variable task difficulty, worker skill, and worker intention. IEEE Trans Knowl Data Eng 27:794\u2013809","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"6825_CR57","doi-asserted-by":"crossref","unstructured":"Dasgupta A, Ghosh A (2013) Crowdsourced judgement elicitation with endogenous proficiency. In: Proceedings of the 22Nd International Conference on World Wide Web, WWW\u20192013, ACM, pp. 319\u2013330","DOI":"10.1145\/2488388.2488417"},{"issue":"10","key":"6825_CR58","doi-asserted-by":"publisher","first-page":"1040","DOI":"10.14778\/2336664.2336676","volume":"5","author":"X Liu","year":"2012","unstructured":"Liu X, Lu M, Ooi BC, Shen Y, Wu S, Zhang M (2012) CDAS: a crowdsourcing data analytics system. Proc VLDB Endow 5(10):1040\u20131051","journal-title":"Proc VLDB Endow"},{"key":"6825_CR59","doi-asserted-by":"publisher","DOI":"10.1561\/9781680832136","volume-title":"Crowdsourcing in Computer Vision","author":"A Kovashka","year":"2016","unstructured":"Kovashka A, Russakovsky O, Fei-Fei L (2016) Crowdsourcing in Computer Vision. Now Publishers Inc., Hanover, MA, USA"},{"key":"6825_CR60","doi-asserted-by":"crossref","unstructured":"Noronha J, Hysen E, Zhang H, Gajos KZ (2011) Platemate: crowdsourcing nutritional analysis from food photographs. In: Proceedings of the 24th Annual ACM Symposium on User Interface Software and Technology, UIST\u20192011, pp. 1\u201312","DOI":"10.1145\/2047196.2047198"},{"issue":"2","key":"6825_CR61","doi-asserted-by":"publisher","first-page":"36","DOI":"10.1080\/08956308.2020.1707007","volume":"63","author":"JH Paik","year":"2020","unstructured":"Paik JH, Scholl M, Sergeev R, Randazzo S, Lakhani KR (2020) Innovation contests for high-tech procurement. Res Technol Manag 63(2):36\u201345","journal-title":"Res Technol Manag"},{"issue":"3","key":"6825_CR62","doi-asserted-by":"publisher","first-page":"391","DOI":"10.1007\/s10606-019-09359-8","volume":"28","author":"I Larsen-Ledet","year":"2019","unstructured":"Larsen-Ledet I, Korsgaard H (2019) Territorial functioning in collaborative writing. Comput Support Cooper Work (CSCW) 28(3):391\u2013433","journal-title":"Comput Support Cooper Work (CSCW)"},{"key":"6825_CR63","unstructured":"Zaidan OF, Callison-Burch C (2011) Crowdsourcing translation: professional quality from non-professionals. In: ACL2011, ACL\u20192011. Association for Computational Linguistics, pp. 1220\u20131229"},{"issue":"1","key":"6825_CR64","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s12879-021-06212-6","volume":"21","author":"KM Tahlil","year":"2021","unstructured":"Tahlil KM, Obiezu-Umeh C, Gbajabiamila T, Nwaozuru U, Oladele D, Musa AZ, Idigbe I, Okwuzu J, David AN, Bamidele TA et al (2021) A designathon to co-create community-driven hiv self-testing services for nigerian youth: findings from a participatory event. BMC Infect Dis 21(1):1\u20138","journal-title":"BMC Infect Dis"},{"issue":"14","key":"6825_CR65","doi-asserted-by":"publisher","first-page":"1758","DOI":"10.1049\/iet-map.2019.0953","volume":"14","author":"M Kolakowski","year":"2020","unstructured":"Kolakowski M (2020) Automatic radio map creation in a fingerprinting-based ble\/uwb localisation system. IET Microw Antennas Propag 14(14):1758\u20131765","journal-title":"IET Microw Antennas Propag"},{"key":"6825_CR66","doi-asserted-by":"crossref","unstructured":"Chai C, Fan J, Li G, Wang J, Zheng Y (2019) Crowdsourcing database systems: overview and challenges. In: 35th IEEE International Conference on Data Engineering, ICDE 2019, Macao, China, April 8-11, 2019, pp. 2052\u20132055","DOI":"10.1109\/ICDE.2019.00237"},{"key":"6825_CR67","doi-asserted-by":"crossref","unstructured":"Ipeirotis PG, Paritosh PK (2011) Managing crowdsourced human computation: a tutorial. In: Proceedings of the 20th international conference companion on World wide web, WWW \u20192011. ACM, pp. 287\u2013288","DOI":"10.1145\/1963192.1963314"},{"key":"6825_CR68","doi-asserted-by":"crossref","unstructured":"Bansal P, Eickhoff C, Hofmann T (2016) Active content-based crowdsourcing task selection. In: Proceedings of the 25th ACM International Conference on Information and Knowledge Management, CIKM\u20192016, pp. 529\u2013538","DOI":"10.1145\/2983323.2983716"},{"issue":"1","key":"6825_CR69","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1287\/opre.2013.1235","volume":"62","author":"DR Karger","year":"2014","unstructured":"Karger DR, Oh S, Shah D (2014) Budget-optimal task allocation for reliable crowdsourcing systems. Oper Res 62(1):1\u201324","journal-title":"Oper Res"},{"key":"6825_CR70","unstructured":"Tran-Thanh L, Huynh TD, Rosenfeld A, Ramchurn SD, Jennings NR (2014) Budgetfix: budget limited crowdsourcing for interdependent task allocation with quality guarantees. In: AAMAS2014, AAMAS\u20192014, pp. 477\u2013484"},{"key":"6825_CR71","doi-asserted-by":"crossref","unstructured":"Tran-Thanh L, Huynh TD, Rosenfeld A, Ramchurn SD, Jennings NR (2015) Crowdsourcing complex workflows under budget constraints. In: AAAI2015, AAAI\u20192015, pp. 1298\u20131304","DOI":"10.1609\/aaai.v29i1.9338"},{"key":"6825_CR72","unstructured":"Yu H, Miao C, Shen Z, Leung C (2015) Quality and budget aware task allocation for spatial crowdsourcing. In: AAMAS2015, AAMAS\u20192015, pp. 1689\u20131690"},{"issue":"4","key":"6825_CR73","doi-asserted-by":"publisher","first-page":"30:1","DOI":"10.1145\/3119930","volume":"24","author":"U Gadiraju","year":"2017","unstructured":"Gadiraju U, Fetahu B, Kawase R, Siehndel P, Dietze S (2017) Using worker self-assessments for competence-based pre-selection in crowdsourcing microtasks. ACM Trans Comput Hum Interact 24(4):30:1-30:26","journal-title":"ACM Trans Comput Hum Interact"},{"key":"6825_CR74","doi-asserted-by":"publisher","unstructured":"Li T (2019) Solving mysteries with the wisdom of crowds: a modularized pipeline and context slices. In: Conference Companion Publication of the 2019 on Computer Supported Cooperative Work and Social Computing, CSCW\u201919. Association for Computing Machinery, New York, NY, USA, pp. 68\u201371 https:\/\/doi.org\/10.1145\/3311957.3361862","DOI":"10.1145\/3311957.3361862"},{"key":"6825_CR75","doi-asserted-by":"crossref","unstructured":"Law E, Zhang H (2011) Towards large-scale collaborative planning: Answering high-level search queries using human computation. In: AAAI2011, pp. 1210\u20131215","DOI":"10.1609\/aaai.v25i1.8092"},{"key":"6825_CR76","doi-asserted-by":"crossref","unstructured":"Kittur A, Smus B, Khamkar S, Kraut RE (2011) Crowdforge: crowdsourcing complex work. In: Proceedings of the 24th annual ACM symposium on User interface software and technology, UIST\u20192011, ACM pp. 43\u201352","DOI":"10.1145\/2047196.2047202"},{"key":"6825_CR77","doi-asserted-by":"publisher","first-page":"625","DOI":"10.1007\/978-1-4614-8806-4_49","volume-title":"Handbook of Human Computation","author":"P Dai","year":"2013","unstructured":"Dai P (2013) Constructing crowdsourced workflows. Handbook of Human Computation. Springer, New York, pp 625\u2013632"},{"key":"6825_CR78","doi-asserted-by":"publisher","first-page":"52","DOI":"10.1016\/j.artint.2013.06.002","volume":"202","author":"P Dai","year":"2013","unstructured":"Dai P, Lin CH, Mausam Weld DS (2013) Pomdp-based control of workflows for crowdsourcing. Artif Intell 202:52\u201385","journal-title":"Artif Intell"},{"issue":"8","key":"6825_CR79","doi-asserted-by":"publisher","first-page":"85","DOI":"10.1145\/2791285","volume":"58","author":"MS Bernstein","year":"2015","unstructured":"Bernstein MS, Little G, Miller RC, Hartmann B, Ackerman MS, Karger DR, Crowell D, Panovich K (2015) Soylent: a word processor with a crowd inside. Commun ACM 58(8):85\u201394","journal-title":"Commun ACM"},{"key":"6825_CR80","doi-asserted-by":"publisher","first-page":"453","DOI":"10.1007\/978-3-319-05813-9_30","volume-title":"Database Systems for Advanced Applications, DASFAA\u20192014","author":"J Feng","year":"2014","unstructured":"Feng J, Li G, Wang H, Feng J (2014) Incremental quality inference in crowdsourcing. Database Systems for Advanced Applications, DASFAA\u20192014. Springer, New York, pp 453\u2013467"},{"key":"6825_CR81","doi-asserted-by":"crossref","unstructured":"Salek M, Bachrach Y, Key P (2013) Hotspotting: a probabilistic graphical model for image object localization through crowdsourcing. In: Proceedings of the Twenty-Seventh AAAI Conference on Artificial Intelligence, AAAI\u20192013. AAAI Press, pp. 1156\u20131162","DOI":"10.1609\/aaai.v27i1.8465"},{"key":"6825_CR82","unstructured":"Li H, Yu B, Zhou D (2013) Error rate bounds in crowdsourcing models. arXiv preprint arXiv:1307.2674"},{"key":"6825_CR83","unstructured":"Gao C, Zhou D (2016) Minimax optimal convergence rates for estimating ground truth from crowdsourced labels. Statistics"},{"key":"6825_CR84","unstructured":"Zhang Y, Chen X, Zhou D, Jordan MI (2014) Spectral methods meet em: a provably optimal algorithm for crowdsourcing. In: Proceedings of the 27th International Conference on Neural Information Processing Systems, NIPS\u20192014, pp. 1260\u20131268"}],"container-title":["Neural Computing and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-021-06825-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00521-021-06825-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-021-06825-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,10,17]],"date-time":"2023-10-17T18:04:32Z","timestamp":1697565872000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00521-021-06825-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,1,17]]},"references-count":84,"journal-issue":{"issue":"31","published-print":{"date-parts":[[2023,11]]}},"alternative-id":["6825"],"URL":"https:\/\/doi.org\/10.1007\/s00521-021-06825-7","relation":{},"ISSN":["0941-0643","1433-3058"],"issn-type":[{"value":"0941-0643","type":"print"},{"value":"1433-3058","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,1,17]]},"assertion":[{"value":"18 May 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"4 December 2021","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 January 2022","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}]}}