{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,23]],"date-time":"2025-10-23T20:57:27Z","timestamp":1761253047695,"version":"3.41.0"},"reference-count":45,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2016,1,6]],"date-time":"2016-01-06T00:00:00Z","timestamp":1452038400000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Ann Math Artif Intell"],"published-print":{"date-parts":[[2016,9]]},"DOI":"10.1007\/s10472-015-9492-4","type":"journal-article","created":{"date-parts":[[2016,1,6]],"date-time":"2016-01-06T19:51:31Z","timestamp":1452109891000},"page":"45-72","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":12,"title":["How many crowdsourced workers should a requester hire?"],"prefix":"10.1007","volume":"78","author":[{"given":"Arthur","family":"Carvalho","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Stanko","family":"Dimitrov","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kate","family":"Larson","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2016,1,6]]},"reference":[{"issue":"8","key":"9492_CR1","doi-asserted-by":"crossref","first-page":"58","DOI":"10.1145\/1378704.1378719","volume":"51","author":"L von Ahn","year":"2008","unstructured":"von Ahn, L., Dabbish, L.: Designing games with a purpose. Commun. ACM 51(8), 58\u201367 (2008)","journal-title":"Commun. ACM"},{"key":"9492_CR2","doi-asserted-by":"crossref","unstructured":"Armstrong, J.S.: Combining Forecasts. In: Armstrong, J.S. (ed.) Principles of Forecasting: A Handbook for Researchers and Practitioners, pp 1\u201319. Kluwer Academic Publishers (2001)","DOI":"10.1007\/978-0-306-47630-3_1"},{"key":"9492_CR3","unstructured":"Bacon, D.F., Chen, Y., Kash, I., Parkes, D.C., Rao, M., Sridharan, M.: Predicting your own effort. In: Proceedings of the 11th International conference on autonomous agents and multiagent systems, pp 695\u2013702 (2012)"},{"issue":"1","key":"9492_CR4","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1002\/jae.659","volume":"18","author":"J Bai","year":"2003","unstructured":"Bai, J., Perron, P.: Computation and analysis of multiple structural change models. J. Appl. Econ. 18(1), 1\u201322 (2003)","journal-title":"J. Appl. Econ."},{"issue":"1","key":"9492_CR5","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1177\/1745691610393980","volume":"6","author":"MD Buhrmester","year":"2011","unstructured":"Buhrmester, M.D., Kwang, T., Gosling, S.D.: Amazon\u2019s mechanical turk: a new source of inexpensive, yet high-quality, data. Perspect. Psychol. Sci. 6(1), 3\u20135 (2011)","journal-title":"Perspect. Psychol. Sci."},{"issue":"1","key":"9492_CR6","doi-asserted-by":"crossref","first-page":"86","DOI":"10.1017\/S193029750000320X","volume":"10","author":"A Carvalho","year":"2015","unstructured":"Carvalho, A.: Tailored proper scoring rules elicit decision weights. Judgment and Decision Making 10(1), 86\u201396 (2015)","journal-title":"Judgment and Decision Making"},{"key":"9492_CR7","unstructured":"Carvalho, A., Dimitrov, S., Larson, K.: Inducing honest reporting without observing outcomes: an application to the peer-review process (2013). arXiv preprint arXiv: 1309.3197"},{"issue":"1","key":"9492_CR8","doi-asserted-by":"crossref","first-page":"77","DOI":"10.1145\/2692375.2692384","volume":"13","author":"A Carvalho","year":"2014","unstructured":"Carvalho, A., Dimitrov, S., Larson, K.: The output-agreement method induces honest behavior in the presence of social projection. ACM SIGecom Exchanges 13(1), 77\u201381 (2014)","journal-title":"ACM SIGecom Exchanges"},{"key":"9492_CR9","doi-asserted-by":"crossref","unstructured":"Carvalho, A., Dimitrov, S., Larson, K.: A study on the influence of the number of mturkers on the quality of the aggregate output. In: Bulling, N. (ed.) Multi-agent systems, lecture notes in computer science, vol. 8953, pp 285\u2013300. Springer (2015)","DOI":"10.1007\/978-3-319-17130-2_19"},{"key":"9492_CR10","unstructured":"Carvalho, A., Larson, K.: Sharing a reward based on peer evaluations. In: Proceedings of the 9th International conference on autonomous agents and multiagent systems, pp 1455\u20131456 (2010)"},{"key":"9492_CR11","unstructured":"Carvalho, A., Larson, K.: A truth serum for sharing rewards. In: Proceedings of the 10th International conference on autonomous agents and multiagent systems, pp 635\u2013642 (2011)"},{"issue":"3","key":"9492_CR12","doi-asserted-by":"crossref","first-page":"253","DOI":"10.1287\/deca.1120.0247","volume":"9","author":"A Carvalho","year":"2012","unstructured":"Carvalho, A., Larson, K.: Sharing rewards among strangers based on peer evaluations. Decis. Anal. 9(3), 253\u2013273 (2012)","journal-title":"Decis. Anal."},{"key":"9492_CR13","unstructured":"Carvalho, A., Larson, K.: A consensual linear opinion pool. In: Proceedings of the 23rd International Joint Conference on Artificial Intelligence, pp 2518\u20132524 (2013)"},{"key":"9492_CR14","doi-asserted-by":"crossref","unstructured":"Chen, Y., Chu, C.H., Mullen, T., Pennock, D.M.: Information markets vs. opinion pools: an empirical comparison. In: Proceedings of the 6th ACM Conference on Electronic Commerce, pp 58\u201367 (2005)","DOI":"10.1145\/1064009.1064016"},{"key":"9492_CR15","doi-asserted-by":"crossref","first-page":"40","DOI":"10.1016\/j.dss.2014.05.010","volume":"65","author":"CM Chiu","year":"2014","unstructured":"Chiu, C.M., Liang, T.P., Turban, E.: What can crowdsourcing do for decision support. Decis. Support. Syst. 65, 40\u201349 (2014)","journal-title":"Decis. Support. Syst."},{"issue":"4","key":"9492_CR16","doi-asserted-by":"crossref","first-page":"559","DOI":"10.1016\/0169-2070(89)90012-5","volume":"5","author":"RT Clemen","year":"1989","unstructured":"Clemen, R.T.: Combining forecasts: a review and annotated bibliography. Int. J. Forecast. 5(4), 559\u2013583 (1989)","journal-title":"Int. J. Forecast."},{"key":"9492_CR17","doi-asserted-by":"crossref","unstructured":"Dietterich, T.G.: Ensemble methods in machine learning. In: Multiple classifier systems, lecture notes in computer science, vol. 1857, pp 1\u201315. Springer (2000)","DOI":"10.1007\/3-540-45014-9_1"},{"key":"9492_CR18","doi-asserted-by":"crossref","unstructured":"Gao, X.A., Mao, A., Chen, Y.: Trick or treat: putting peer prediction to the test. In: Proceedings of the 1st workshop on crowdsourcing and online behavioral experiments (2013)","DOI":"10.1145\/2600057.2602865"},{"issue":"10","key":"9492_CR19","doi-asserted-by":"crossref","first-page":"993","DOI":"10.1109\/34.58871","volume":"12","author":"LK Hansen","year":"1990","unstructured":"Hansen, L.K., Salamon, P.: Neural Network Ensembles. IEEE Trans. Pattern Anal. Mach. Intell. 12(10), 993\u20131001 (1990)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"issue":"1","key":"9492_CR20","doi-asserted-by":"crossref","first-page":"107","DOI":"10.1023\/A:1022058209073","volume":"5","author":"R Hanson","year":"2003","unstructured":"Hanson, R.: Combinatorial information market design. Inf. Syst. Front. 5(1), 107\u2013119 (2003)","journal-title":"Inf. Syst. Front."},{"key":"9492_CR21","doi-asserted-by":"crossref","unstructured":"Ho, C.J., Vaughan, J.W.: Online task assignment in crowdsourcing markets. In: Proceedings of the 26th AAAI conference on artificial intelligence, pp 45\u201351 (2012)","DOI":"10.1609\/aaai.v26i1.8120"},{"key":"9492_CR22","doi-asserted-by":"crossref","unstructured":"Huang, S.W., Fu, W.T.: Enhancing reliability using peer consistency evaluation in human computation. In: Proceedings of the 2013 conference on computer supported cooperative work, pp 639\u2013648 (2013)","DOI":"10.1145\/2441776.2441847"},{"issue":"2","key":"9492_CR23","doi-asserted-by":"crossref","first-page":"16","DOI":"10.1145\/1869086.1869094","volume":"17","author":"PG Ipeirotis","year":"2010","unstructured":"Ipeirotis, P.G.: Analyzing the amazon mechanical turk marketplace. XRDS Crossroads: The ACM Magazine for Students 17(2), 16\u201321 (2010)","journal-title":"XRDS Crossroads: The ACM Magazine for Students"},{"issue":"2","key":"9492_CR24","doi-asserted-by":"crossref","first-page":"402","DOI":"10.1007\/s10618-013-0306-1","volume":"28","author":"PG Ipeirotis","year":"2014","unstructured":"Ipeirotis, P.G., Provost, F., Sheng, V.S., Wang, J.: Repeated labeling using multiple noisy labelers. Data Min. Knowl. Disc 28(2), 402\u2013441 (2014)","journal-title":"Data Min. Knowl. Disc"},{"key":"9492_CR25","unstructured":"Lin, C.H., Weld, D.S.: Dynamically switching between synergistic workflows for crowdsourcing. In: Proceedings of the 26th AAAI conference on artificial intelligence, pp 132\u2013133 (2012)"},{"key":"9492_CR26","doi-asserted-by":"crossref","unstructured":"Marge, M., Banerjee, S., Rudnicky, A.I.: Using the amazon mechanical turk for transcription of spoken language. In: Proceedings of the 2010 IEEE International conference on acoustics speech and signal processing, pp 5270\u20135273 (2010)","DOI":"10.1109\/ICASSP.2010.5494979"},{"issue":"1","key":"9492_CR27","doi-asserted-by":"crossref","first-page":"1","DOI":"10.3758\/s13428-011-0124-6","volume":"44","author":"W Mason","year":"2012","unstructured":"Mason, W., Suri, S.: Conducting behavioral research on amazon\u2019s mechanical turk. Behav. Res. Methods 44(1), 1\u201323 (2012)","journal-title":"Behav. Res. Methods"},{"key":"9492_CR28","unstructured":"Neruda: P.: 100 Love Sonnets. Exile (2007)"},{"key":"9492_CR29","doi-asserted-by":"crossref","unstructured":"Oshiro, T.M., Perez, P.S., Baranauskas, J.A.: How many trees in a random forest In: Perner, P. (ed.) Machine learning and data mining in pattern recognition, Lecture notes in computer science, vol. 7376, pp 154\u2013168. Springer, Berlin (2012)","DOI":"10.1007\/978-3-642-31537-4_13"},{"issue":"5","key":"9492_CR30","doi-asserted-by":"crossref","first-page":"411","DOI":"10.1017\/S1930297500002205","volume":"5","author":"G Paolacci","year":"2010","unstructured":"Paolacci, G., Chandler, J., Ipeirotis, P.G.: Running experiments on amazon mechanical turk. Judgment and Decision making 5(5), 411\u2013419 (2010)","journal-title":"Judgment and Decision making"},{"key":"9492_CR31","doi-asserted-by":"crossref","unstructured":"Plous, S.: The Psychology of Judgment and Decision Making. Mcgraw-Hill Book Company (1993)","DOI":"10.1037\/e412982005-012"},{"key":"9492_CR32","doi-asserted-by":"crossref","unstructured":"Quinn, A.J., Bederson, B.B.: Human computation: a survey and taxonomy of a growing field. In: Proceedings of the 2011 SIGCHI conference on human factors in computing systems, pp 1403\u20131412 (2011)","DOI":"10.1145\/1978942.1979148"},{"key":"9492_CR33","doi-asserted-by":"crossref","first-page":"28","DOI":"10.1016\/j.dss.2014.05.009","volume":"65","author":"J Ren","year":"2014","unstructured":"Ren, J., Nickerson, J.V., Mason, W., Sakamoto, Y., Graber, B.: Increasing the crowd\u2019s capacity to create: how alternative generation affects the diversity, relevance and effectiveness of generated ads. Decis. Support. Syst. 65, 28\u201339 (2014)","journal-title":"Decis. Support. Syst."},{"issue":"336","key":"9492_CR34","doi-asserted-by":"crossref","first-page":"783","DOI":"10.1080\/01621459.1971.10482346","volume":"66","author":"LJ Savage","year":"1971","unstructured":"Savage, L.J.: Elicitation of personal probabilities and expectations. J. Am. Stat. Assoc 66(336), 783\u2013801 (1971)","journal-title":"J. Am. Stat. Assoc"},{"issue":"1","key":"9492_CR35","doi-asserted-by":"crossref","first-page":"43","DOI":"10.1023\/A:1009957816843","volume":"1","author":"R Selten","year":"1998","unstructured":"Selten, R.: Axiomatic characterization of the quadratic scoring rule. Exp. Econ 1(1), 43\u201362 (1998)","journal-title":"Exp. Econ"},{"key":"9492_CR36","doi-asserted-by":"crossref","unstructured":"Shaw, A.D., Horton, J.J., Chen, D.L.: Designing incentives for inexpert human raters. In: Proceedings of the ACM 2011 conference on computer supported cooperative work, pp 275\u2013284 (2011)","DOI":"10.1145\/1958824.1958865"},{"key":"9492_CR37","doi-asserted-by":"crossref","unstructured":"Sheng, V.S., Provost, F., Ipeirotis, P.G.: Get another label? Improving data quality and data mining using multiple, noisy labelers. In: Proceedings of the 14th International conference on knowledge discovery and data mining, pp 614\u2013622 (2008)","DOI":"10.1145\/1401890.1401965"},{"key":"9492_CR38","doi-asserted-by":"crossref","unstructured":"Snow, R., O\u2019Connor, B., Jurafsky, D., Ng, A.Y.: Cheap and fast\u2014But is it good? Evaluating non-expert annotations for natural language tasks. In: Proceedings of the conference on empirical methods in natural language processing, pp 254\u2013263 (2008)","DOI":"10.3115\/1613715.1613751"},{"key":"9492_CR39","unstructured":"Taylor, J., Taylor, A., Greenaway, K.: Little Ann and Other Poems. Nabu Press (2010)"},{"key":"9492_CR40","doi-asserted-by":"crossref","unstructured":"Tran-Thanh, L., Stein, S., Rogers, A., Jennings, N.R.: Efficient crowdsourcing of unknown experts using multi-armed bandits. In: Proceedings of the 20th European conference on artificial intelligence, pp 768\u2013773 (2012)","DOI":"10.3233\/978-1-61499-098-7-768"},{"issue":"3","key":"9492_CR41","doi-asserted-by":"crossref","first-page":"167","DOI":"10.1287\/deca.1030.0008","volume":"1","author":"RL Winkler","year":"2004","unstructured":"Winkler, R.L., Clemen, R.T.: Multiple experts vs. multiple methods: combining correlation assessments. Decis. Anal 1(3), 167\u2013176 (2004)","journal-title":"Decis. Anal"},{"issue":"5","key":"9492_CR42","doi-asserted-by":"crossref","first-page":"751","DOI":"10.1175\/1520-0450(1968)007<0751:PA>2.0.CO;2","volume":"7","author":"RL Winkler","year":"1968","unstructured":"Winkler, R.L., Murphy, A.H.: \u201cGood\u201d Probability Assessors. J. Appl. Meteorol 7(5), 751\u2013758 (1968)","journal-title":"J. Appl. Meteorol"},{"key":"9492_CR43","doi-asserted-by":"crossref","unstructured":"Yuen, M.C., King, I., Leung, K.S.: A survey of crowdsourcing systems. In: Proceedings of IEEE 3rd International Conference on Social Computing, pp 766\u2013773 (2011)","DOI":"10.1109\/PASSAT\/SocialCom.2011.203"},{"issue":"2","key":"9492_CR44","doi-asserted-by":"crossref","first-page":"1","DOI":"10.18637\/jss.v007.i02","volume":"7","author":"A Zeileis","year":"2002","unstructured":"Zeileis, A., Leisch, F., Hornik, K., Kleiber, C.: strucchange: an R package for testing for structural change in linear regression models. J. Stat. Softw 7(2), 1\u201338 (2002)","journal-title":"J. Stat. Softw"},{"key":"9492_CR45","doi-asserted-by":"crossref","unstructured":"Zhang, H., Horvitz, E., Parkes, D.: Automated workflow synthesis. In: Proceedings of the 27th AAAI conference on artificial intelligence, pp 1020\u20131026 (2013)","DOI":"10.1609\/aaai.v27i1.8681"}],"container-title":["Annals of Mathematics and Artificial Intelligence"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10472-015-9492-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10472-015-9492-4\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10472-015-9492-4","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,1]],"date-time":"2025-06-01T02:09:08Z","timestamp":1748743748000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10472-015-9492-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,1,6]]},"references-count":45,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2016,9]]}},"alternative-id":["9492"],"URL":"https:\/\/doi.org\/10.1007\/s10472-015-9492-4","relation":{},"ISSN":["1012-2443","1573-7470"],"issn-type":[{"type":"print","value":"1012-2443"},{"type":"electronic","value":"1573-7470"}],"subject":[],"published":{"date-parts":[[2016,1,6]]}}}