{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,25]],"date-time":"2026-04-25T11:20:41Z","timestamp":1777116041958,"version":"3.51.4"},"reference-count":51,"publisher":"Springer Science and Business Media LLC","issue":"12","license":[{"start":{"date-parts":[[2022,9,22]],"date-time":"2022-09-22T00:00:00Z","timestamp":1663804800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,9,22]],"date-time":"2022-09-22T00:00:00Z","timestamp":1663804800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62031003"],"award-info":[{"award-number":["62031003"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Shenzhen Polytechnic Youth Innovation Project","award":["6019310007K0"],"award-info":[{"award-number":["6019310007K0"]}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62072380"],"award-info":[{"award-number":["62072380"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Knowl Inf Syst"],"published-print":{"date-parts":[[2022,12]]},"DOI":"10.1007\/s10115-022-01759-5","type":"journal-article","created":{"date-parts":[[2022,9,22]],"date-time":"2022-09-22T04:03:31Z","timestamp":1663819411000},"page":"3235-3263","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Self-paced annotations of crowd workers"],"prefix":"10.1007","volume":"64","author":[{"given":"Xiangping","family":"Kang","sequence":"first","affiliation":[]},{"given":"Guoxian","family":"Yu","sequence":"additional","affiliation":[]},{"given":"Carlotta","family":"Domeniconi","sequence":"additional","affiliation":[]},{"given":"Jun","family":"Wang","sequence":"additional","affiliation":[]},{"given":"Wei","family":"Guo","sequence":"additional","affiliation":[]},{"given":"Yazhou","family":"Ren","sequence":"additional","affiliation":[]},{"given":"Xiayan","family":"Zhang","sequence":"additional","affiliation":[]},{"given":"Lizhen","family":"Cui","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2022,9,22]]},"reference":[{"key":"1759_CR1","doi-asserted-by":"crossref","unstructured":"Bengio Y, Louradour J, Collobert R, Weston J (2009) Curriculum learning. In: International conference on machine learning, 41\u201348","DOI":"10.1145\/1553374.1553380"},{"key":"1759_CR2","volume-title":"Bayesian theory","author":"JM Bernardo","year":"2009","unstructured":"Bernardo JM, Smith AF (2009) Bayesian theory, vol 405. Wiley, Hoboken"},{"issue":"7307","key":"1759_CR3","doi-asserted-by":"publisher","first-page":"756","DOI":"10.1038\/nature09304","volume":"466","author":"S Cooper","year":"2010","unstructured":"Cooper S, Khatib F, Treuille A, Barbero J, Lee J, Beenen M, Leaver-Fay A, Baker D, Popovi\u0107 Z et al (2010) Predicting protein structures with a multiplayer online game. Nature 466(7307):756\u2013760","journal-title":"Nature"},{"issue":"1","key":"1759_CR4","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10115-017-1057-x","volume":"53","author":"V Crescenzi","year":"2017","unstructured":"Crescenzi V, Fernandes AA, Merialdo P, Paton NW (2017) Crowdsourcing for data management. Knowl Info Sys 53(1):1\u201341","journal-title":"Knowl Info Sys"},{"issue":"1","key":"1759_CR5","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3148148","volume":"51","author":"F Daniel","year":"2018","unstructured":"Daniel F, Kucherbaev P, Cappiello C, Benatallah B, Allahbakhsh M (2018) Quality control in crowdsourcing: a survey of quality attributes, assessment techniques and assurance actions. ACM Comput Surv (CSUR) 51(1):1\u201340","journal-title":"ACM Comput Surv (CSUR)"},{"key":"1759_CR6","doi-asserted-by":"crossref","unstructured":"Demartini G, Difallah DE, Cudr\u00e9-Mauroux P (2012) Zencrowd: leveraging probabilistic reasoning and crowdsourcing techniques for large-scale entity linking. In: Proceedings of the 21st International Conference on World Wide Web, 469\u2013478","DOI":"10.1145\/2187836.2187900"},{"issue":"1","key":"1759_CR7","first-page":"1","volume":"7","author":"J Dem\u0161ar","year":"2006","unstructured":"Dem\u0161ar J (2006) Statistical comparisons of classifiers over multiple data sets. J Mach Learn Resear 7(1):1\u201330","journal-title":"J Mach Learn Resear"},{"key":"1759_CR8","doi-asserted-by":"crossref","unstructured":"Deng J, Dong W, Socher R, Li L-J, Li K, Li F-F (2009) Imagenet: A large-scale hierarchical image database. In: IEEE conference on computer vision and pattern recognition, pp 248\u2013255. IEEE","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"1759_CR9","doi-asserted-by":"crossref","unstructured":"Fan J, Li G, Ooi BC, Tan K-l, Feng J (2015) icrowd: an adaptive crowdsourcing framework. In: ACM Conference on management of data, 1015\u20131030","DOI":"10.1145\/2723372.2750550"},{"key":"1759_CR10","unstructured":"Griffin G, Holub A, Perona P (2007) Caltech-256 object category dataset"},{"key":"1759_CR11","doi-asserted-by":"crossref","unstructured":"Han G, Tu J, Yu G, Wang J, Domeniconi C (2020) Crowdsourcing with multiple-source knowledge transfer. In: International joint conference on artificial intelligence, 2908\u20132914","DOI":"10.24963\/ijcai.2020\/402"},{"key":"1759_CR12","doi-asserted-by":"crossref","unstructured":"Ho C-J, Vaughan J (2012) Online task assignment in crowdsourcing markets. In Proceedings of the AAAI conference on artificial intelligence, 45\u201351","DOI":"10.1609\/aaai.v26i1.8120"},{"key":"1759_CR13","doi-asserted-by":"crossref","unstructured":"Hu Z, Zhang J (2018) A novel strategy for active task assignment in crowd labeling. In International joint conference on artificial intelligence, 1538\u20131545","DOI":"10.24963\/ijcai.2018\/213"},{"key":"1759_CR14","doi-asserted-by":"crossref","unstructured":"Ipeirotis PG, Provost F, Wang J (2010) Quality management on amazon mechanical turk. In: Proceedings of the ACM SIGKDD workshop on human computation, 64\u201367","DOI":"10.1145\/1837885.1837906"},{"key":"1759_CR15","doi-asserted-by":"crossref","unstructured":"Jiang L, Meng D, Mitamura T, Hauptmann A (2014) Easy samples first: self-paced reranking for zero-example multimedia search. In: ACM International conference on multimedia, 547\u2013556","DOI":"10.1145\/2647868.2654918"},{"key":"1759_CR16","doi-asserted-by":"crossref","unstructured":"Kang X, Yu G, Domeniconi C, Wang J, Guo W, Ren Y, Cui L (2021) Crowdsourcing with self-paced workers. In: IEEE International conference on data mining, 280\u2013289","DOI":"10.1109\/ICDM51629.2021.00038"},{"key":"1759_CR17","doi-asserted-by":"crossref","unstructured":"Karger DR, Oh S, Shah D (2011) Budget-optimal crowdsourcing using low-rank matrix approximations. In 2011 49th Annual allerton conference on communication, control and computing, 284\u2013291","DOI":"10.1109\/Allerton.2011.6120180"},{"key":"1759_CR18","doi-asserted-by":"crossref","unstructured":"Kazai G, Kamps J, Milic-Frayling N (2011) Worker types and personality traits in crowdsourcing relevance labels. In: ACM International conference on information and knowledge management, 1941\u20131944","DOI":"10.1145\/2063576.2063860"},{"key":"1759_CR19","doi-asserted-by":"crossref","unstructured":"Khan AR, Garcia-Molina H (2017) Crowddqs: Dynamic question selection in crowdsourcing systems. In: ACM conference on management of data, 1447\u20131462","DOI":"10.1145\/3035918.3064055"},{"key":"1759_CR20","doi-asserted-by":"crossref","unstructured":"Korycki \u0141, Krawczyk B (2017) Combining active learning and self-labeling for data stream mining. In: International conference on computer recognition systems, 481\u2013490","DOI":"10.1007\/978-3-319-59162-9_50"},{"key":"1759_CR21","doi-asserted-by":"crossref","unstructured":"Kovashka A, Russakovsky O, Fei-Fei L, Grauman K (2016) Crowdsourcing in computer vision. arXiv preprint arXiv:1611.02145","DOI":"10.1561\/9781680832136"},{"key":"1759_CR22","unstructured":"Kumar MP, Packer B, Koller D (2010) Self-paced learning for latent variable models. In Advances in neural information processing systems, 1189\u20131197"},{"issue":"2","key":"1759_CR23","doi-asserted-by":"publisher","first-page":"108","DOI":"10.1038\/nbt.2495","volume":"31","author":"KR Lakhani","year":"2013","unstructured":"Lakhani KR, Boudreau KJ, Loh P-R, Backstrom L, Baldwin C, Lonstein E, Lydon M, MacCormack A, Arnaout RA, Guinan EC (2013) Prize-based contests can provide solutions to computational biology problems. Nat Biotech 31(2):108\u2013111","journal-title":"Nat Biotech"},{"key":"1759_CR24","doi-asserted-by":"crossref","unstructured":"Lang K (1995) Newsweeder: learning to filter netnews. In Proceedings of the 12th International conference on machine learning, 331\u2013339","DOI":"10.1016\/B978-1-55860-377-6.50048-7"},{"key":"1759_CR25","doi-asserted-by":"crossref","unstructured":"Li G, Zheng Y, Fan J, Wang J, Cheng R (2017) Crowdsourced data management: Overview and challenges. In ACM SIGMOD conference on management of data, 1711\u20131716","DOI":"10.1145\/3035918.3054776"},{"issue":"10","key":"1759_CR26","doi-asserted-by":"publisher","first-page":"1040","DOI":"10.14778\/2336664.2336676","volume":"5","author":"X Liu","year":"2012","unstructured":"Liu X, Lu M, Ooi BC, Shen Y, Wu S, Zhang M (2012) Cdas: a crowdsourcing data analytics system. Proceed VLDB Endowm 5(10):1040\u20131051","journal-title":"Proceed VLDB Endowm"},{"key":"1759_CR27","doi-asserted-by":"crossref","unstructured":"Ma F, Li Y, Li Q, Qiu M, Gao J, Zhi S, Su L, Zhao B, Ji H, Han J (2015) Faitcrowd: fine grained truth discovery for crowdsourced data aggregation. In: Proceedings of ACM SIGKDD International conference on knowledge discovery and data mining, 745\u2013754","DOI":"10.1145\/2783258.2783314"},{"issue":"2","key":"1759_CR28","doi-asserted-by":"publisher","first-page":"109","DOI":"10.14778\/2535568.2448944","volume":"6","author":"A Marcus","year":"2012","unstructured":"Marcus A, Karger D, Madden S, Miller R, Oh S (2012) Counting with the crowd. Proceed VLDB Endowm 6(2):109\u2013120","journal-title":"Proceed VLDB Endowm"},{"key":"1759_CR29","doi-asserted-by":"publisher","first-page":"319","DOI":"10.1016\/j.ins.2017.05.043","volume":"414","author":"D Meng","year":"2017","unstructured":"Meng D, Zhao Q, Jiang L (2017) A theoretical understanding of self-paced learning. Info Sci 414:319\u2013328","journal-title":"Info Sci"},{"issue":"2","key":"1759_CR30","doi-asserted-by":"publisher","first-page":"271","DOI":"10.1007\/s10115-014-0790-7","volume":"45","author":"J Muhammadi","year":"2015","unstructured":"Muhammadi J, Rabiee HR, Hosseini A (2015) A unified statistical framework for crowd labeling. Knowl Info Sys 45(2):271\u2013294","journal-title":"Knowl Info Sys"},{"issue":"1","key":"1759_CR31","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10115-018-1235-5","volume":"60","author":"L Nassar","year":"2019","unstructured":"Nassar L, Karray F (2019) Overview of the crowdsourcing process. Knowl Info Sys 60(1):1\u201324","journal-title":"Knowl Info Sys"},{"issue":"12","key":"1759_CR32","doi-asserted-by":"publisher","first-page":"1565","DOI":"10.1038\/nbt1206-1565","volume":"24","author":"WS Noble","year":"2006","unstructured":"Noble WS (2006) What is a support vector machine? Nat Biotech 24(12):1565\u20131567","journal-title":"Nat Biotech"},{"key":"1759_CR33","doi-asserted-by":"crossref","unstructured":"Que X, Ren Y, Zhou J, Xu Z (2017) Regularized multi-source matrix factorization for diagnosis of alzheimer\u2019s disease. In: International conference on neural information processing, 463\u2013473. Springer","DOI":"10.1007\/978-3-319-70087-8_49"},{"issue":"4","key":"1759_CR34","first-page":"1297","volume":"11","author":"VC Raykar","year":"2010","unstructured":"Raykar VC, Yu S, Zhao LH, Valadez GH, Florin C, Bogoni L, Moy L (2010) Learning from crowds. J Mach Learn Resear 11(4):1297\u20131322","journal-title":"J Mach Learn Resear"},{"key":"1759_CR35","unstructured":"Simonyan K, Zisserman A (2014) Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556"},{"issue":"7","key":"1759_CR36","doi-asserted-by":"publisher","first-page":"2521","DOI":"10.1007\/s10115-020-01475-y","volume":"62","author":"F Tao","year":"2020","unstructured":"Tao F, Jiang L, Li C (2020) Label similarity-based weighted soft majority voting and pairing for crowdsourcing. Knowl Info Sys 62(7):2521\u20132538","journal-title":"Knowl Info Sys"},{"issue":"4","key":"1759_CR37","doi-asserted-by":"publisher","first-page":"1341","DOI":"10.1007\/s10115-019-01386-7","volume":"62","author":"J Tu","year":"2020","unstructured":"Tu J, Yu G, Domeniconi C, Wang J, Xiao G, Guo M (2020) Multi-label crowd consensus via joint matrix factorization. Knowl Info Sys 62(4):1341\u20131369","journal-title":"Knowl Info Sys"},{"issue":"1","key":"1759_CR38","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3421712","volume":"15","author":"J Tu","year":"2021","unstructured":"Tu J, Yu G, Wang J, Domeniconi C, Guo M, Zhang X (2021) Crowdwt: Crowdsourcing via joint modeling of workers and tasks. ACM Trans Knowl Discov Data 15(1):1\u201324","journal-title":"ACM Trans Knowl Discov Data"},{"key":"1759_CR39","doi-asserted-by":"crossref","unstructured":"Tu J, Yu G, Wang J, Domeniconi C, Zhang X (2020b) Attention-aware answers of the crowd. In SIAM International conference on data mining, 451\u2013459","DOI":"10.1137\/1.9781611976236.51"},{"key":"1759_CR40","unstructured":"Wah C, Branson S, Welinder P, Perona P, Belongie S (2011) The caltech-ucsd birds-200-2011 dataset"},{"issue":"1","key":"1759_CR41","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1007\/s10579-012-9176-1","volume":"47","author":"A Wang","year":"2013","unstructured":"Wang A, Hoang CDV, Kan M-Y (2013) Perspectives on crowdsourcing annotations for natural language processing. Lang Resour Evaluat 47(1):9\u201331","journal-title":"Lang Resour Evaluat"},{"key":"1759_CR42","doi-asserted-by":"crossref","unstructured":"Wang J, Kraska T, Franklin MJ, Feng J (2012) Crowder: crowdsourcing entity resolution. arXiv preprint arXiv:1208.1927","DOI":"10.14778\/2350229.2350263"},{"key":"1759_CR43","doi-asserted-by":"crossref","unstructured":"Wang W, Guo X-Y, Li S-Y, Jiang Y, Zhou Z-H (2017) Obtaining high-quality label by distinguishing between easy and hard items in crowdsourcing. In: International joint conference on artificial intelligence, 2964\u20132970","DOI":"10.24963\/ijcai.2017\/413"},{"key":"1759_CR44","unstructured":"Whitehill J, Wu T-f, Bergsma J, Movellan J, Ruvolo P (2009) Whose vote should count more: Optimal integration of labels from labelers of unknown expertise. In: Advances in neural information processing systems, 2035\u20132043"},{"key":"1759_CR45","doi-asserted-by":"crossref","unstructured":"Xie Q, Luong M-T, Hovy E, Le QV (2020) Self-training with noisy student improves imagenet classification. In: IEEE Conference on computer vision and pattern recognition, 10687\u201310698","DOI":"10.1109\/CVPR42600.2020.01070"},{"key":"1759_CR46","doi-asserted-by":"publisher","first-page":"134","DOI":"10.1016\/j.neucom.2018.04.075","volume":"309","author":"W Xu","year":"2018","unstructured":"Xu W, Liu W, Huang X, Yang J, Qiu S (2018) Multi-modal self-paced learning for image classification. Neurocomputing 309:134\u2013144","journal-title":"Neurocomputing"},{"issue":"4","key":"1759_CR47","doi-asserted-by":"publisher","first-page":"1448","DOI":"10.1109\/TNNLS.2020.2984729","volume":"32","author":"G Yu","year":"2021","unstructured":"Yu G, Tu J, Wang J, Domeniconi C, Zhang X (2021) Active multilabel crowd consensus. IEEE Trans Neur Netw Learn Sys 32(4):1448\u20131459","journal-title":"IEEE Trans Neur Netw Learn Sys"},{"issue":"5","key":"1759_CR48","doi-asserted-by":"publisher","first-page":"1675","DOI":"10.1109\/TNNLS.2017.2677468","volume":"29","author":"J Zhang","year":"2017","unstructured":"Zhang J, Sheng VS, Li T, Wu X (2017) Improving crowdsourced label quality using noise correction. IEEE Trans Neur Netw Learn Sys 29(5):1675\u20131688","journal-title":"IEEE Trans Neur Netw Learn Sys"},{"key":"1759_CR49","doi-asserted-by":"crossref","unstructured":"Zhang X, Shi H, Li Y, Liang W (2017b) Spglad: A self-paced learning-based crowdsourcing classification model. In Pacific-Asia conference on knowledge discovery and data mining, 189\u2013201","DOI":"10.1007\/978-3-319-67274-8_17"},{"issue":"4","key":"1759_CR50","doi-asserted-by":"publisher","first-page":"361","DOI":"10.14778\/3025111.3025118","volume":"10","author":"Y Zheng","year":"2016","unstructured":"Zheng Y, Li G, Cheng R (2016) Docs: a domain-aware crowdsourcing system using knowledge bases. Proceed VLDB Endow 10(4):361\u2013372","journal-title":"Proceed VLDB Endow"},{"key":"1759_CR51","doi-asserted-by":"crossref","unstructured":"Zheng Y, Wang J, Li G, Cheng R, Feng J (2015) Qasca: a quality-aware task assignment system for crowdsourcing applications. In: ACM SIGMOD conference on management of data, 1031\u20131046","DOI":"10.1145\/2723372.2749430"}],"container-title":["Knowledge and Information Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-022-01759-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10115-022-01759-5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-022-01759-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,10,21]],"date-time":"2022-10-21T06:20:56Z","timestamp":1666333256000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10115-022-01759-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,9,22]]},"references-count":51,"journal-issue":{"issue":"12","published-print":{"date-parts":[[2022,12]]}},"alternative-id":["1759"],"URL":"https:\/\/doi.org\/10.1007\/s10115-022-01759-5","relation":{},"ISSN":["0219-1377","0219-3116"],"issn-type":[{"value":"0219-1377","type":"print"},{"value":"0219-3116","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,9,22]]},"assertion":[{"value":"17 January 2022","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"28 August 2022","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"3 September 2022","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"22 September 2022","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}