{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,30]],"date-time":"2025-12-30T17:50:43Z","timestamp":1767117043100,"version":"3.37.3"},"reference-count":33,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2019,6,13]],"date-time":"2019-06-13T00:00:00Z","timestamp":1560384000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2019,6,13]],"date-time":"2019-06-13T00:00:00Z","timestamp":1560384000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Data Min Knowl Disc"],"published-print":{"date-parts":[[2019,11]]},"DOI":"10.1007\/s10618-019-00637-z","type":"journal-article","created":{"date-parts":[[2019,6,13]],"date-time":"2019-06-13T15:03:48Z","timestamp":1560438228000},"page":"1625-1673","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["More for less: adaptive labeling payments in online labor markets"],"prefix":"10.1007","volume":"33","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4501-3414","authenticated-orcid":false,"given":"Tomer","family":"Geva","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2400-3762","authenticated-orcid":false,"given":"Maytal","family":"Saar-Tsechansky","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2953-9598","authenticated-orcid":false,"given":"Harel","family":"Lustiger","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2019,6,13]]},"reference":[{"key":"637_CR1","unstructured":"Abe N, Mamitsuka H (1998) Query learning strategies using boosting and bagging. In: Proceedings of the international conference on machine learning (ICML). Morgan Kaufmann, pp 1\u20139"},{"issue":"2","key":"637_CR7","doi-asserted-by":"crossref","first-page":"403","DOI":"10.1287\/isre.2016.0633","volume":"27","author":"DL Chen","year":"2016","unstructured":"Chen DL, Horton JJ (2016) Research note\u2014are online labor markets spot markets for tasks? A field experiment on the behavioral response to wage cuts. Inf Syst Res 27(2):403\u2013423","journal-title":"Inf Syst Res"},{"key":"637_CR10","doi-asserted-by":"crossref","first-page":"52","DOI":"10.1016\/j.artint.2013.06.002","volume":"202","author":"P Dai","year":"2013","unstructured":"Dai P, Lin CH, Mausam M, Weld DS (2013) Pomdp-based control of workflows for crowdsourcing. Artif Intell 202:52\u201385","journal-title":"Artif Intell"},{"key":"637_CR11","doi-asserted-by":"crossref","unstructured":"Dalvi N, Dasgupta A, Kumar R, Rastogi V (2013) Aggregating crowdsourced binary ratings. In: Proceedings of the 22nd international conference on world wide web. ACM, New York, pp 285\u2013294","DOI":"10.1145\/2488388.2488414"},{"key":"637_CR12","doi-asserted-by":"crossref","unstructured":"Downs JS, Holbrook MB, Sheng S, Cranor LF (2010) Are your participants gaming the system? Screening mechanical Turk workers. In: Proceedings of the SIGCHI conference on human factors in computing systems. New York: ACM, pp 2399\u20132402","DOI":"10.1145\/1753326.1753688"},{"key":"637_CR13","doi-asserted-by":"crossref","unstructured":"Feng D, Besana S, Zajac R (2009) Acquiring high quality non-expert knowledge from on-demand workforce. In: Proceedings of the 2009 workshop on the people\u2019s web meets NLP: collaboratively constructed semantic resources. Association for Computational Linguistics, Stroudsburg, pp 51\u201356","DOI":"10.3115\/1699765.1699773"},{"issue":"2","key":"637_CR15","doi-asserted-by":"crossref","first-page":"402","DOI":"10.1007\/s10618-013-0306-1","volume":"28","author":"PG Ipeirotis","year":"2014","unstructured":"Ipeirotis PG, Provost F, Sheng VS, Wang J (2014) Repeated labeling using multiple noisy labelers. Data Min Knowl Disc 28(2):402\u2013441","journal-title":"Data Min Knowl Disc"},{"key":"637_CR17","unstructured":"Karger DR, Oh S, Shah D (2011) Iterative learning for reliable crowdsourcing systems. In: Proceedings of advances in neural information processing systems: 25th annual conference on neural information processing, December 12\u201314"},{"issue":"1","key":"637_CR18","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1287\/opre.2013.1235","volume":"62","author":"DR Karger","year":"2014","unstructured":"Karger DR, Oh S, Shah D (2014) Budget-optimal task allocation for reliable crowdsourcing systems. Oper Res 62(1):1\u201324","journal-title":"Oper Res"},{"key":"637_CR19","doi-asserted-by":"crossref","first-page":"165","DOI":"10.1007\/978-3-642-20161-5_17","volume-title":"Advances in information retrieval","author":"G Kazai","year":"2011","unstructured":"Kazai G (2011) In search of quality in crowdsourcing for search engine evaluation. In: Clough P et al (eds) Advances in information retrieval. Springer, Berlin Heidelberg, pp 165\u2013176"},{"issue":"2","key":"637_CR20","doi-asserted-by":"crossref","first-page":"138","DOI":"10.1007\/s10791-012-9205-0","volume":"16","author":"G Kazai","year":"2013","unstructured":"Kazai G, Kamps J, Milic-Frayling N (2013) An analysis of human factors and label accuracy in crowdsourcing relevance judgments. Inf Retrieval 16(2):138\u2013178","journal-title":"Inf Retrieval"},{"issue":"1","key":"637_CR23","doi-asserted-by":"crossref","first-page":"71","DOI":"10.1007\/s10994-013-5424-x","volume":"95","author":"D Kong","year":"2014","unstructured":"Kong D, Saar-Tsechansky M (2014) Collaborative information acquisition for data-driven decisions. Mach Learn 95(1):71\u201386","journal-title":"Mach Learn"},{"key":"637_CR24","unstructured":"Kumar A, Lease M (2011) Modeling annotator accuracies for supervised learning. In: Proceedings of the workshop on crowdsourcing for search and data mining (CSDM), at the fourth ACM international conference on web search and data mining (WSDM), pp 19\u201322"},{"key":"637_CR27","unstructured":"Lee D, Hosanagar K, Nair H (2013) The effect of advertising content on consumer engagement: evidence from Facebook (working paper). Available at SSRN 2290802"},{"key":"637_CR28","doi-asserted-by":"crossref","unstructured":"Lewis D, Gale W (1994) A sequential algorithm for training text classifiers. In: Proceedings of the ACM SIGIR conference on research and development in information retrieval. ACM\/Springer, pp 3\u201312","DOI":"10.1007\/978-1-4471-2099-5_1"},{"key":"637_CR29","unstructured":"Lichman M (2013) UCI machine learning repository. University of California, School of Information and Computer Science, Irvine. http:\/\/archive.ics.uci.edu\/ml"},{"key":"637_CR30","doi-asserted-by":"crossref","unstructured":"Lin CH, Weld DS (2014) To re (label), or not to re (label). In: Second AAAI conference on human computation and crowdsourcing","DOI":"10.1609\/hcomp.v2i1.13167"},{"key":"637_CR31","unstructured":"Lin CH, Mausam M, Weld DS (2012) Crowdsourcing control: moving beyond multiple choice. In: UAI"},{"key":"637_CR32","doi-asserted-by":"crossref","unstructured":"Lin CH, Mausam M, Weld DS (2016) Re-active learning: active learning with relabeling. In: AAAI, pp 1845\u20131852","DOI":"10.1609\/aaai.v30i1.10315"},{"issue":"1","key":"637_CR33","doi-asserted-by":"crossref","first-page":"1","DOI":"10.3758\/s13428-011-0124-6","volume":"44","author":"W Mason","year":"2012","unstructured":"Mason W, Suri S (2012) Conducting behavioral research on Amazon\u2019s mechanical Turk. Behav Res Methods 44(1):1\u201323","journal-title":"Behav Res Methods"},{"issue":"2","key":"637_CR34","doi-asserted-by":"crossref","first-page":"100","DOI":"10.1145\/1809400.1809422","volume":"11","author":"W Mason","year":"2010","unstructured":"Mason W, Watts DJ (2010) Financial incentives and the performance of crowds. ACM SIGKDD Explor Newsl 11(2):100\u2013108","journal-title":"ACM SIGKDD Explor Newsl"},{"issue":"5","key":"637_CR35","doi-asserted-by":"crossref","first-page":"411","DOI":"10.1017\/S1930297500002205","volume":"5","author":"G Paolacci","year":"2010","unstructured":"Paolacci G, Chandler J, Ipeirotis PG (2010) Running experiments on amazon mechanical Turk. Judgm Decis Mak 5(5):411\u2013419","journal-title":"Judgm Decis Mak"},{"issue":"11","key":"637_CR37","first-page":"1297","volume":"1","author":"VC Raykar","year":"2010","unstructured":"Raykar VC, Yu S, Zhao LH, Valadez GH, Florin C, Bogoni L, Moy L (2010) Learning from crowds. J Mach Learn Res 1(11):1297\u20131322","journal-title":"J Mach Learn Res"},{"issue":"12","key":"637_CR38","doi-asserted-by":"crossref","first-page":"1428","DOI":"10.1016\/j.patrec.2013.05.012","volume":"34","author":"F Rodrigues","year":"2013","unstructured":"Rodrigues F, Pereira F, Ribeiro B (2013) Learning from multiple annotators: distinguishing good from random labelers. Pattern Recogn Lett 34(12):1428\u20131436","journal-title":"Pattern Recogn Lett"},{"key":"637_CR39","first-page":"17","volume":"11","author":"J Rogstadius","year":"2011","unstructured":"Rogstadius J, Kostakos V, Kittur A, Smus B, Laredo J, Vukovic M (2011) An assessment of intrinsic and extrinsic motivation on task performance in crowdsourcing markets. ICWSM 11:17\u201321","journal-title":"ICWSM"},{"issue":"2","key":"637_CR40","doi-asserted-by":"crossref","first-page":"153","DOI":"10.1023\/B:MACH.0000011806.12374.c3","volume":"54","author":"M Saar-Tsechansky","year":"2004","unstructured":"Saar-Tsechansky M, Provost F (2004) Active sampling for class probability estimation and ranking. Mach Learn 54(2):153\u2013178","journal-title":"Mach Learn"},{"key":"637_CR42","doi-asserted-by":"crossref","unstructured":"Sheng VS, Provost F, Ipeirotis PG (2008) Get another label? Improving data quality and data mining using multiple, noisy labelers. In: Proceedings of the 14th ACM SIGKDD international conference on knowledge discovery and data mining, pp 614\u2013622","DOI":"10.1145\/1401890.1401965"},{"key":"637_CR44","doi-asserted-by":"crossref","first-page":"137","DOI":"10.1287\/isre.2016.0661","volume":"28","author":"J Wang","year":"2017","unstructured":"Wang J, Ipeirotis PG, Provost F (2017) Cost-effective quality assurance in crowd labeling. Inf Syst Res 28:137\u2013158","journal-title":"Inf Syst Res"},{"key":"637_CR45","unstructured":"Wauthier FL, Jordan MI (2011) Bayesian bias mitigation for crowdsourcing. In: Bartlett P, Pereira F, Shawe-Taylor J, Zemel R (eds.) Advances in neural information processing systems (NIPS), pp 1800\u20131808"},{"key":"637_CR46","doi-asserted-by":"crossref","first-page":"121","DOI":"10.1007\/978-1-4419-1428-6_1796","volume-title":"Encyclopedia of the sciences of learning","author":"L Yang","year":"2012","unstructured":"Yang L, Carbonell J (2012) Adaptive Proactive learning with cost-reliability trade-off. In: Seel NM (ed) Encyclopedia of the sciences of learning. Springer, New York, pp 121\u2013127"},{"issue":"5","key":"637_CR47","doi-asserted-by":"crossref","first-page":"1095","DOI":"10.1109\/TCYB.2014.2344674","volume":"45","author":"Jing Zhang","year":"2015","unstructured":"Zhang Jing, Xindong Wu, Sheng Victor S (2015) Active learning with imbalanced multiple noisy labeling. IEEE Trans Cybern 45(5):1095\u20131107","journal-title":"IEEE Trans Cybern"},{"issue":"4","key":"637_CR48","doi-asserted-by":"crossref","first-page":"543","DOI":"10.1007\/s10462-016-9491-9","volume":"46","author":"Jing Zhang","year":"2016","unstructured":"Zhang Jing, Xindong Wu, Sheng Victor S (2016) Learning from crowdsourced labeled data: a survey. Artif Intell Rev 46(4):543\u2013576","journal-title":"Artif Intell Rev"},{"key":"637_CR49","unstructured":"Zhou D, Basu S, Mao Y, Platt JC (2012) Learning from the wisdom of crowds by minimax entropy. In: Advances in neural information processing systems, pp 2204\u20132212"}],"container-title":["Data Mining and Knowledge Discovery"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10618-019-00637-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10618-019-00637-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10618-019-00637-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,9,17]],"date-time":"2023-09-17T12:16:01Z","timestamp":1694952961000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10618-019-00637-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,6,13]]},"references-count":33,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2019,11]]}},"alternative-id":["637"],"URL":"https:\/\/doi.org\/10.1007\/s10618-019-00637-z","relation":{},"ISSN":["1384-5810","1573-756X"],"issn-type":[{"type":"print","value":"1384-5810"},{"type":"electronic","value":"1573-756X"}],"subject":[],"published":{"date-parts":[[2019,6,13]]},"assertion":[{"value":"15 December 2017","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"16 April 2019","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 June 2019","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}