{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,5]],"date-time":"2026-01-05T22:32:21Z","timestamp":1767652341768,"version":"3.40.3"},"publisher-location":"Cham","reference-count":50,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783319664347"},{"type":"electronic","value":"9783319664354"}],"license":[{"start":{"date-parts":[[2017,1,1]],"date-time":"2017-01-01T00:00:00Z","timestamp":1483228800000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2017]]},"DOI":"10.1007\/978-3-319-66435-4_4","type":"book-chapter","created":{"date-parts":[[2017,9,27]],"date-time":"2017-09-27T11:51:34Z","timestamp":1506513094000},"page":"70-95","source":"Crossref","is-referenced-by-count":7,"title":["Crowdsourcing Technology to Support Academic Research"],"prefix":"10.1007","author":[{"given":"Matthias","family":"Hirth","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jason","family":"Jacques","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Peter","family":"Rodgers","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ognjen","family":"Scekic","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Michael","family":"Wybrow","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2017,9,28]]},"reference":[{"issue":"3","key":"4_CR1","doi-asserted-by":"crossref","first-page":"272","DOI":"10.1057\/sj.2012.48","volume":"28","author":"AA Adams","year":"2015","unstructured":"Adams, A.A., Ferryman, J.M.: The future of video analytics for surveillance and its ethical implications. Secur. J. 28(3), 272\u2013289 (2015)","journal-title":"Secur. J."},{"issue":"1","key":"4_CR2","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1111\/j.1744-6570.1991.tb00688.x","volume":"44","author":"MR Barrick","year":"1991","unstructured":"Barrick, M.R., Mount, M.K.: The big five personality dimensions and job performance: a meta-analysis. Person. Psychol. 44(1), 1\u201326 (1991)","journal-title":"Person. Psychol."},{"issue":"3","key":"4_CR3","doi-asserted-by":"crossref","first-page":"387","DOI":"10.1348\/096317905X26994","volume":"78","author":"C Bertua","year":"2005","unstructured":"Bertua, C., Anderson, N., Salgado, J.F.: The predictive validity of cognitive ability tests: a UK meta-analysis. J. Occup. Organ. Psychol. 78(3), 387\u2013409 (2005)","journal-title":"J. Occup. Organ. Psychol."},{"key":"4_CR4","doi-asserted-by":"crossref","unstructured":"Breslav, S., Khan, A., Hornb\u00e6k, K.: Mimic: visual analytics of online micro-interactions. In: Proceedings of the 2014 International Working Conference on Advanced Visual Interfaces (AVI 2014), pp. 245\u2013252, NY, USA. ACM, New York (2014)","DOI":"10.1145\/2598153.2598168"},{"issue":"2","key":"4_CR5","doi-asserted-by":"crossref","first-page":"111","DOI":"10.13169\/workorgalaboglob.4.2.0111","volume":"4","author":"B Caraway","year":"2010","unstructured":"Caraway, B.: Online labour markets: an inquiry into oDesk providers. Work Organ. Labour Globalisation 4(2), 111\u2013125 (2010)","journal-title":"Work Organ. Labour Globalisation"},{"issue":"1","key":"4_CR6","doi-asserted-by":"crossref","first-page":"112","DOI":"10.3758\/s13428-013-0365-7","volume":"46","author":"J Chandler","year":"2014","unstructured":"Chandler, J., Mueller, P., Paolacci, G.: Nonna\u00efvet\u00e9 among amazon mechanical turk workers: consequences and solutions for behavioral researchers. Behav. Res. Methods 46(1), 112\u2013130 (2014)","journal-title":"Behav. Res. Methods"},{"issue":"3","key":"4_CR7","doi-asserted-by":"crossref","first-page":"e57410","DOI":"10.1371\/journal.pone.0057410","volume":"8","author":"MJ Crump","year":"2013","unstructured":"Crump, M.J., McDonnell, J.V., Gureckis, T.M.: Evaluating amazon\u2019s mechanical turk as a tool for experimental behavioral research. PloS one 8(3), e57410 (2013)","journal-title":"PloS one"},{"key":"4_CR8","doi-asserted-by":"crossref","unstructured":"Difallah, D.E., Demartini, G., Cudr\u00e9-Mauroux, P.: Mechanical cheat: spamming schemes and adversarial techniques on crowdsourcing platforms. In: CrowdSearch, pp. 26\u201330 (2012)","DOI":"10.1145\/2187836.2187900"},{"issue":"4","key":"4_CR9","doi-asserted-by":"crossref","first-page":"86","DOI":"10.1145\/1924421.1924442","volume":"54","author":"A Doan","year":"2011","unstructured":"Doan, A., Ramakrishnan, R., Halevy, A.Y.: Crowdsourcing systems on the World-Wide Web. Commun. ACM 54(4), 86\u201396 (2011)","journal-title":"Commun. ACM"},{"key":"4_CR10","doi-asserted-by":"crossref","unstructured":"Eickhoff, C., Harris, C.G., de Vries, A.P., Srinivasan, P.: Quality through flow and immersion: gamifying crowdsourced relevance assessments. In: Proceedings of the 35th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2012), pp. 871\u2013880, NY, USA. ACM, New York (2012)","DOI":"10.1145\/2348283.2348400"},{"key":"4_CR11","doi-asserted-by":"crossref","unstructured":"Elkhodr, M., Shahrestani, S., Cheung, H.: A semantic obfuscation technique for the internet of things. In: 2014 IEEE International Conference on Communications Workshops (ICC), pp. 448\u2013453, June 2014","DOI":"10.1109\/ICCW.2014.6881239"},{"issue":"1","key":"4_CR12","first-page":"143","volume":"32","author":"A Felstiner","year":"2011","unstructured":"Felstiner, A.: Working the crowd: employment and labor law in the crowdsourcing industry. Berkeley J. Employ. Labor Law 32(1), 143 (2011)","journal-title":"Berkeley J. Employ. Labor Law"},{"key":"4_CR13","unstructured":"Ferreira, D., Kostakos, V., Dey, A.K.: AWARE: mobile context instrumentation framework. Front. ICT 2 (2015). http:\/\/journal.frontiersin.org\/article\/10.3389\/fict.2015.00006\/full"},{"issue":"3","key":"4_CR14","doi-asserted-by":"crossref","first-page":"427","DOI":"10.1016\/j.ijindorg.2003.10.004","volume":"22","author":"A Glazer","year":"2004","unstructured":"Glazer, A.: Motivating devoted workers. Int. J. Ind. Organ. 22(3), 427\u2013440 (2004)","journal-title":"Int. J. Ind. Organ."},{"issue":"3","key":"4_CR15","doi-asserted-by":"crossref","first-page":"213","DOI":"10.1002\/bdm.1753","volume":"26","author":"JK Goodman","year":"2013","unstructured":"Goodman, J.K., Cryder, C.E., Cheema, A.: Data collection in a flat world: the strengths and weaknesses of mechanical turk samples. J. Behav. Decis. Making 26(3), 213\u2013224 (2013)","journal-title":"J. Behav. Decis. Making"},{"issue":"7","key":"4_CR16","doi-asserted-by":"crossref","first-page":"623","DOI":"10.1016\/j.acn.2006.05.007","volume":"21","author":"CT Gualtieri","year":"2006","unstructured":"Gualtieri, C.T., Johnson, L.G.: Reliability and validity of a computerized neurocognitive test battery, CNS vital signs. Arch. Clin. Neuropsychol. 21(7), 623\u2013643 (2006)","journal-title":"Arch. Clin. Neuropsychol."},{"key":"4_CR17","series-title":"IFIP Advances in Information and Communication Technology","doi-asserted-by":"publisher","first-page":"237","DOI":"10.1007\/978-3-319-18621-4_16","volume-title":"Privacy and Identity Management for the Future Internet in the Age of Globalisation","author":"M Hartswood","year":"2015","unstructured":"Hartswood, M., Jirotka, M., Chenu-Abente, R., Hume, A., Giunchiglia, F., Martucci, L.A., Fischer-H\u00fcbner, S.: Privacy for peer profiling in collective adaptive systems. In: Camenisch, J., Fischer-H\u00fcbner, S., Hansen, M. (eds.) Privacy and Identity 2014. IAICT, vol. 457, pp. 237\u2013252. Springer, Cham (2015). doi: 10.1007\/978-3-319-18621-4_16"},{"key":"4_CR18","doi-asserted-by":"crossref","unstructured":"Hauber, M., Bachmann, A., Budde, M., Beigl, M.: jActivity: supporting mobile web developers with HTML5\/JavaScript based human activity recognition. In: Proceedings of the 12th International Conference on Mobile and Ubiquitous Multimedia (MUM 2013), pp. 45:1\u201345:2, NY, USA. ACM, New York (2013)","DOI":"10.1145\/2541831.2541873"},{"key":"4_CR19","doi-asserted-by":"crossref","unstructured":"Heckman, J.J., Smith, J.A., Taber, C.: What do bureaucrats do? The effects of performance standards and bureaucratic preferences on acceptance into the JTPA program. In: Advances in the Study of Entrepreneurship Innovation and Economic Growth, vol. 7, pp. 191\u2013217 (1996)","DOI":"10.3386\/w5535"},{"key":"4_CR20","doi-asserted-by":"crossref","unstructured":"Hirth, M., Ho\u00dffeld, T., Tran-Gia, P.: Anatomy of a crowdsourcing platform \u2013 using the example of Microworkers.com. In: Workshop on Future Internet and Next Generation Networks (FINGNet), Seoul, Korea, June 2011","DOI":"10.1109\/IMIS.2011.89"},{"key":"4_CR21","doi-asserted-by":"crossref","unstructured":"Hirth, M., Scheuring, S., Ho\u00dffeld, T., Schwartz, C., Tran-Gia, P.: Predicting result quality in crowdsourcing using application layer monitoring. In: 2014 Fifth International Conference on Communications and Electronics (ICCE). IEEE (2014)","DOI":"10.1109\/CCE.2014.6916756"},{"issue":"2","key":"4_CR22","doi-asserted-by":"crossref","first-page":"541","DOI":"10.1109\/TMM.2013.2291663","volume":"16","author":"T Hossfeld","year":"2014","unstructured":"Hossfeld, T., Keimel, C., Hirth, M., Gardlo, B., Habigt, J., Diepold, K., Tran-Gia, P.: Best practices for QoE crowdtesting: QoE assessment with crowdsourcing. Trans. Multimed. 16(2), 541\u2013558 (2014)","journal-title":"Trans. Multimed."},{"key":"4_CR23","doi-asserted-by":"crossref","unstructured":"Irani, L.C., Silberman, M.S.: Turkopticon: interrupting worker invisibility in amazon mechanical turk. In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems (CHI 2013), pp. 611\u2013620, NY, USA. ACM, New York (2013)","DOI":"10.1145\/2470654.2470742"},{"key":"4_CR24","doi-asserted-by":"crossref","unstructured":"Jacques, J.T., Kristensson, P.O.: Crowdsourcing a HIT: measuring workers\u2019 pre-task interactions on microtask markets. In: First AAAI Conference on Human Computation and Crowdsourcing, November 2013","DOI":"10.1609\/hcomp.v1i1.13085"},{"key":"4_CR25","doi-asserted-by":"crossref","unstructured":"Kazai, G., Zitouni, I.: Quality management in crowdsourcing using gold judges behavior. In: Proceedings of the Ninth ACM International Conference on Web Search and Data Mining, pp. 267\u2013276. ACM (2016)","DOI":"10.1145\/2835776.2835835"},{"issue":"1","key":"4_CR26","doi-asserted-by":"crossref","first-page":"349","DOI":"10.1109\/TVCG.2015.2467451","volume":"22","author":"S Kieffer","year":"2016","unstructured":"Kieffer, S., Dwyer, T., Marriott, K., Wybrow, M.: Hola: human-like orthogonal network layout. IEEE Trans. Vis. Comput. Graph. 22(1), 349\u2013358 (2016)","journal-title":"IEEE Trans. Vis. Comput. Graph."},{"key":"4_CR27","doi-asserted-by":"crossref","unstructured":"Kittur, A., Nickerson, J.V., Bernstein, M., Gerber, E., Shaw, A., Zimmerman, J., Lease, M., Horton, J.: The future of crowd work. In: Proceedings of the 2013 Conference on Computer Supported Cooperative Work (CSCW 2013), pp. 1301\u20131318. ACM (2013)","DOI":"10.1145\/2441776.2441923"},{"key":"4_CR28","doi-asserted-by":"crossref","unstructured":"Lebreton, P., Hupont, I., M\u00e4ki, T., Skodras, E., Hirth, M.: Eye tracker in the wild: studying the delta between what is said and measured in a crowdsourcing experiment. In: Proceedings of the Fourth International Workshop on Crowdsourcing for Multimedia, pp. 3\u20138. ACM (2015)","DOI":"10.1145\/2810188.2810192"},{"key":"4_CR29","doi-asserted-by":"crossref","unstructured":"Lebreton, P., M\u00e4ki, T., Skodras, E., Hupont, I., Hirth, M.: Bridging the gap between eye tracking and crowdsourcing, vol. 9394, pp. 93940W\u201393940W-14 (2015)","DOI":"10.1117\/12.2076745"},{"key":"4_CR30","doi-asserted-by":"crossref","unstructured":"Little, G.: Exploring iterative and parallel human computation processes. In: Proceedings of the 28th International Conference on Human Factors in Computing Systems, CHI 2010, Extended Abstracts Volume, Atlanta, Georgia, USA, 10\u201315 April 2010, pp. 4309\u20134314 (2010)","DOI":"10.1145\/1753846.1754145"},{"key":"4_CR31","doi-asserted-by":"crossref","unstructured":"Mao, A., Kamar, E., Chen, Y., Horvitz, E., Schwamb, M.E., Lintott, C.J., Smith, A.M.: Volunteering versus work for pay: incentives and tradeoffs in crowdsourcing. In: Hartman, B., Horvitz, E. (eds.) HCOMP. AAAI (2013)","DOI":"10.1609\/hcomp.v1i1.13075"},{"key":"4_CR32","doi-asserted-by":"crossref","unstructured":"Martin, D.B., Hanrahan, B.V., O\u2019Neill, J., Gupta, N.: Being a turker. In: Computer Supported Cooperative Work (CSCW 2014), Baltimore, MD, USA, 15\u201319 February 2014, pp. 224\u2013235 (2014)","DOI":"10.1145\/2531602.2531663"},{"key":"4_CR33","doi-asserted-by":"crossref","unstructured":"Mason, W., Watts, D.J.: Financial incentives and the \u201cperformance of crowds\u201d. In: Proceedings of the ACM SIGKDD Workshop on Human Computation (HCOMP 2009), NY, USA, pp. 77\u201385. ACM, New York (2009)","DOI":"10.1145\/1600150.1600175"},{"key":"4_CR34","doi-asserted-by":"crossref","unstructured":"McDuff, D., el Kaliouby, R., Picard, R.W.: Crowdsourcing facial responses to online videos: extended abstract. In: 2015 International Conference on Affective Computing and Intelligent Interaction (ACII), pp. 512\u2013518, September 2015","DOI":"10.1109\/ACII.2015.7344618"},{"key":"4_CR35","doi-asserted-by":"crossref","unstructured":"McDuff, D., el Kaliouby, R., Picard, R.: Crowdsourced data collection of facial responses. In: Proceedings of the 13th International Conference on Multimodal Interfaces (ICMI 2011), NY, USA pp. 11\u201318. ACM, New York (2011)","DOI":"10.1145\/2070481.2070486"},{"key":"4_CR36","doi-asserted-by":"crossref","unstructured":"Mok, R.K., Li, W., Chang, R.K.: Detecting low-quality crowdtesting workers. In: 2015 IEEE 23rd International Symposium on Quality of Service (IWQoS), pp. 201\u2013206. IEEE (2015)","DOI":"10.1109\/IWQoS.2015.7404734"},{"key":"4_CR37","doi-asserted-by":"crossref","unstructured":"Narayanan, A., Shmatikov, V.: Robust de-anonymization of large sparse datasets. In: IEEE Symposium on Security and Privacy (SP 2008), pp. 111\u2013125. IEEE (2008)","DOI":"10.1109\/SP.2008.33"},{"key":"4_CR38","doi-asserted-by":"crossref","unstructured":"Navalpakkam, V., Churchill, E.: Mouse tracking: measuring and predicting users\u2019 experience of web-based content. In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems (CHI 2012), NY, USA, pp. 2963\u20132972. ACM, New York (2012)","DOI":"10.1145\/2207676.2208705"},{"key":"4_CR39","doi-asserted-by":"crossref","unstructured":"Peer, E., Samat, S., Brandimarte, L., Acquisti, A.: Beyond the turk: an empirical comparison of alternative platforms for online behavioral research. Available at SSRN 2594183, April 2015","DOI":"10.2139\/ssrn.2594183"},{"key":"4_CR40","doi-asserted-by":"crossref","unstructured":"Raddick, M.J., Bracey, G., Gay, P.L., Lintott, C.J., Murray, P., Schawinski, K., Szalay, A.S., Vandenberg, J.: Galaxy zoo: exploring the motivations of citizen science volunteers. Astron. Educ. Rev. 9(1) (2010). http:\/\/portico.org\/Portico\/#!journalAUSimpleView\/tab=HTML?cs=ISSN_15391515?ct=E-Journal%20Content?auId=ark:\/27927\/pgg3ztfdp8z","DOI":"10.3847\/AER2009036"},{"key":"4_CR41","doi-asserted-by":"crossref","unstructured":"Reed, J., Raddick, M.J., Lardner, A., Carney, K.: An exploratory factor analysis of motivations for participating in Zooniverse, a collection of virtual citizen science projects. In: 2013 46th Hawaii International Conference on System Sciences (HICSS), pp. 610\u2013619, January 2013","DOI":"10.1109\/HICSS.2013.85"},{"key":"4_CR42","unstructured":"Richardson, D.W., Gribble, S.D.: Maverick: providing web applications with safe and flexible access to local devices. In: Proceedings of the 2011 USENIX Conference on Web Application Development (2011)"},{"key":"4_CR43","doi-asserted-by":"crossref","unstructured":"Salehi, N., Irani, L.C., Bernstein, M.S., Alkhatib, A., Ogbe, E., Milland, K.: Clickhappier: we are dynamo: overcoming stalling and friction in collective action for crowd workers. In: Proceedings of the 33rd Annual ACM Conference on Human Factors in Computing Systems (CHI 2015), pp. 1621\u20131630. ACM, New York (2015)","DOI":"10.1145\/2702123.2702508"},{"issue":"3","key":"4_CR44","doi-asserted-by":"crossref","first-page":"2","DOI":"10.1109\/MNET.2015.7113217","volume":"29","author":"X Shen","year":"2015","unstructured":"Shen, X.: Mobile crowdsourcing [Editor\u2019s note]. IEEE Netw. 29(3), 2\u20133 (2015)","journal-title":"IEEE Netw."},{"issue":"1","key":"4_CR45","doi-asserted-by":"crossref","first-page":"47","DOI":"10.1007\/s10796-015-9578-x","volume":"18","author":"NH Thuan","year":"2016","unstructured":"Thuan, N.H., Antunes, P., Johnstone, D.: Factors influencing the decision to crowdsource: a systematic literature review. Inf. Syst. Front. 18(1), 47\u201368 (2016)","journal-title":"Inf. Syst. Front."},{"key":"4_CR46","doi-asserted-by":"crossref","unstructured":"V\u00e4\u00e4t\u00e4j\u00e4, H.K., Ahvenainen, M.J., Jaakola, M.S., Olsson, T.D.: Exploring augmented reality for user-generated hyperlocal news content. In: CHI 2013 Extended Abstracts on Human Factors in Computing Systems (CHI EA 2013), NY, USA, pp. 967\u2013972. ACM, New York (2013)","DOI":"10.1145\/2468356.2468529"},{"key":"4_CR47","doi-asserted-by":"crossref","DOI":"10.4159\/harvard.9780674067424","volume-title":"Triumphs of Experience","author":"G Vaillant","year":"2012","unstructured":"Vaillant, G.: Triumphs of Experience. Harvard University Press, Boston (2012)"},{"key":"4_CR48","unstructured":"Vakharia, D., Lease, M.: Beyond AMT: an analysis of crowd work platforms. In: iConference 2015 Proceedings. iSchools, March 2015"},{"issue":"4","key":"4_CR49","first-page":"65:1","volume":"31","author":"HY Wu","year":"2012","unstructured":"Wu, H.Y., Rubinstein, M., Shih, E., Guttag, J., Durand, F., Freeman, W.: Eulerian video magnification for revealing subtle changes in the world. ACM Trans. Graph. 31(4), 65:1\u201365:8 (2012)","journal-title":"ACM Trans. Graph."},{"key":"4_CR50","unstructured":"Xu, P., Ehinger, K.A., Zhang, Y., Finkelstein, A., Kulkarni, S.R., Xiao, J.: TurkerGaze: crowdsourcing saliency with webcam based eye tracking, April 2015. arXiv:1504.06755 [cs]"}],"container-title":["Lecture Notes in Computer Science","Evaluation in the Crowd. Crowdsourcing and Human-Centered Experiments"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-319-66435-4_4","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,8,26]],"date-time":"2023-08-26T12:12:22Z","timestamp":1693051942000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-319-66435-4_4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017]]},"ISBN":["9783319664347","9783319664354"],"references-count":50,"URL":"https:\/\/doi.org\/10.1007\/978-3-319-66435-4_4","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2017]]}}}