{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,25]],"date-time":"2025-06-25T23:10:04Z","timestamp":1750893004992,"version":"3.41.0"},"publisher-location":"Cham","reference-count":46,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783319664347"},{"type":"electronic","value":"9783319664354"}],"license":[{"start":{"date-parts":[[2017,1,1]],"date-time":"2017-01-01T00:00:00Z","timestamp":1483228800000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2017]]},"DOI":"10.1007\/978-3-319-66435-4_3","type":"book-chapter","created":{"date-parts":[[2017,9,27]],"date-time":"2017-09-27T11:51:34Z","timestamp":1506513094000},"page":"27-69","source":"Crossref","is-referenced-by-count":15,"title":["Understanding the Crowd: Ethical and Practical Matters in the Academic Use of Crowdsourcing"],"prefix":"10.1007","author":[{"given":"David","family":"Martin","sequence":"first","affiliation":[]},{"given":"Sheelagh","family":"Carpendale","sequence":"additional","affiliation":[]},{"given":"Neha","family":"Gupta","sequence":"additional","affiliation":[]},{"given":"Tobias","family":"Ho\u00dffeld","sequence":"additional","affiliation":[]},{"given":"Babak","family":"Naderi","sequence":"additional","affiliation":[]},{"given":"Judith","family":"Redi","sequence":"additional","affiliation":[]},{"given":"Ernestasia","family":"Siahaan","sequence":"additional","affiliation":[]},{"given":"Ina","family":"Wechsung","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2017,9,28]]},"reference":[{"key":"3_CR1","doi-asserted-by":"crossref","unstructured":"Antin, J., Shaw, A.: Social desirability bias and self-reports of motivation: a study of Amazon mechanical turk in the US and India. In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems (CHI 2012), pp. 2925\u20132934 (2012)","DOI":"10.1145\/2207676.2208699"},{"key":"3_CR2","doi-asserted-by":"crossref","unstructured":"Bederson, B.B., Quinn, A.J.: Web workers unite! Addressing challenges of online laborers. In: CHI 2011 Extended Abstracts on Human Factors in Computing Systems, pp. 97\u2013106. ACM (2011)","DOI":"10.1145\/1979742.1979606"},{"key":"3_CR3","unstructured":"Berg, J.: Income security in the on-demand economy: findings and policy lessons from a survey of crowdworkers. Comp. Labor Law Policy J. 37(3) (2016)"},{"issue":"1","key":"3_CR4","doi-asserted-by":"crossref","first-page":"68","DOI":"10.1177\/1056492613481245","volume":"23","author":"KA Bourne","year":"2014","unstructured":"Bourne, K.A., Forman, P.J.: Living in a culture of overwork an ethnographic study of flexibility. J. Manag. Inquiry 23(1), 68\u201379 (2014)","journal-title":"J. Manag. Inquiry"},{"key":"3_CR5","doi-asserted-by":"crossref","first-page":"531","DOI":"10.1016\/j.chb.2015.08.031","volume":"54","author":"AM Brawley","year":"2016","unstructured":"Brawley, A.M., Pury, C.L.: Work experiences on MTurk: job satisfaction, turnover, and information sharing. Comput. Hum. Behav. 54, 531\u2013546 (2016)","journal-title":"Comput. Hum. Behav."},{"key":"3_CR6","doi-asserted-by":"crossref","unstructured":"Callison-Burch, C.: Crowd-workers: aggregating information across turkers to help them find higher paying work. In: Second AAAI Conference on Human Computation and Crowdsourcing (2014)","DOI":"10.1609\/hcomp.v2i1.13198"},{"issue":"1","key":"3_CR7","doi-asserted-by":"crossref","first-page":"112","DOI":"10.3758\/s13428-013-0365-7","volume":"46","author":"J Chandler","year":"2014","unstructured":"Chandler, J., Mueller, P., Paolacci, G.: Nonnavet among Amazon mechanical turk workers: consequences and solutions for behavioral researchers. Behav. Res. Methods 46(1), 112\u2013130 (2014)","journal-title":"Behav. Res. Methods"},{"key":"3_CR8","volume-title":"24\/7: Late Capitalism and the Ends of Sleep","author":"J Crary","year":"2013","unstructured":"Crary, J.: 24\/7: Late Capitalism and the Ends of Sleep. Verso Books, New York (2013)"},{"key":"3_CR9","doi-asserted-by":"crossref","DOI":"10.1007\/978-1-4899-2271-7","volume-title":"Intrinsic Motivation and Self-determination in Human Behavior","author":"E Deci","year":"1985","unstructured":"Deci, E., Ryan, R.: Intrinsic Motivation and Self-determination in Human Behavior. Plenum Press, New York (1985)"},{"key":"3_CR10","doi-asserted-by":"crossref","unstructured":"Difallah, D.E., Catasta, M., Demartini, G., Ipeirotis, P.G., Cudr-Mauroux, P.: The dynamics of micro-task crowdsourcing: the case of Amazon MTurk. In: Proceedings of the 24th International Conference on World Wide Web (WWW 2015), pp. 238\u2013247. ACM (2015)","DOI":"10.1145\/2736277.2741685"},{"key":"3_CR11","first-page":"143","volume":"32","author":"A Felstiner","year":"2011","unstructured":"Felstiner, A.: Working the crowd: employment and labor law in the crowdsourcing industry. Berkeley J. Employ. Labor Law 32, 143\u2013203 (2011)","journal-title":"Berkeley J. Employ. Labor Law"},{"key":"3_CR12","unstructured":"Finkin, M.: Beclouded work in historical perspective. Comp. Labor Law Policy J. 37(3) (2016)"},{"issue":"2","key":"3_CR13","doi-asserted-by":"crossref","first-page":"413","DOI":"10.1162\/COLI_a_00057","volume":"37","author":"K Fort","year":"2011","unstructured":"Fort, K., Adda, G., Cohen, K.B.: Amazon mechanical turk: gold mine or coal mine? Comput. Linguist. 37(2), 413\u2013420 (2011)","journal-title":"Comput. Linguist."},{"issue":"4","key":"3_CR14","doi-asserted-by":"crossref","first-page":"331","DOI":"10.1002\/job.322","volume":"26","author":"M Gagn\u00e9","year":"2005","unstructured":"Gagn\u00e9, M., Deci, E.L.: Self-determination theory and work motivation. J. Organ. Behav. 26(4), 331\u2013362 (2005)","journal-title":"J. Organ. Behav."},{"key":"3_CR15","unstructured":"Garfinkel, H.: A conception of and experiments with \u201ctrust\u201d as a condition of concerted stable actions. In: The Production of Reality: Essays and Readings on Social Interaction, pp. 381\u2013392 (1963)"},{"key":"3_CR16","unstructured":"Gupta, N.: An ethnographic study of crowdwork via Amazon mechanical turk in India. Unpublished manuscript (2017). http:\/\/eprints.nottingham.ac.uk\/41062\/"},{"key":"3_CR17","doi-asserted-by":"crossref","unstructured":"Gupta, N., Martin, D., Hanrahan, B.V., O\u2019Neill, J.: Turk-life in India. In: Proceedings of the 18th International Conference on Supporting Group Work, pp. 1\u201311. ACM (2014)","DOI":"10.1145\/2660398.2660403"},{"key":"3_CR18","doi-asserted-by":"crossref","unstructured":"Hanhart, P., Korshunov, P., Ebrahimi, T.: Crowdsourcing evaluation of high dynamic range image compression. In: SPIE Optical Engineering + Applications, p. 92170D. International Society for Optics and Photonics (2014)","DOI":"10.1117\/12.2065560"},{"key":"3_CR19","doi-asserted-by":"crossref","unstructured":"Hanrahan, B.V., Willamowski, J.K., Swaminathan, S., Martin, D.B.: TurkBench: rendering the market for turkers. In: Proceedings of the 33rd Annual ACM Conference on Human Factors in Computing Systems, pp. 1613\u20131616. ACM (2015)","DOI":"10.1145\/2702123.2702279"},{"key":"3_CR20","unstructured":"Health UDo, Services H et al.: The Belmont report. Office for Human Research Protections (OHRP) (1979). Accessed 19 Nov 2008"},{"key":"3_CR21","doi-asserted-by":"crossref","unstructured":"Hirth, M., Hofeld, T., Tran-Gia, P.: Anatomy of a crowdsourcing platform-using the example of microworkers.com. In: 2011 Fifth International Conference on Innovative Mobile and Internet Services in Ubiquitous Computing (IMIS), pp. 322\u2013329. IEEE (2011)","DOI":"10.1109\/IMIS.2011.89"},{"key":"3_CR22","unstructured":"Ipeirotis, P.G.: Demographics of mechanical turk (2010)"},{"issue":"2","key":"3_CR23","doi-asserted-by":"crossref","first-page":"16","DOI":"10.1145\/1869086.1869094","volume":"17","author":"PG Ipeirotis","year":"2010","unstructured":"Ipeirotis, P.G.: Analyzing the Amazon mechanical turk marketplace. XRDS 17(2), 16\u201321 (2010)","journal-title":"XRDS"},{"key":"3_CR24","unstructured":"Ipeirotis, P.: Demographics of mechanical turk: now live! April 2015. http:\/\/www.behind-the-enemy-lines.com\/2015\/04\/demographics-of-mechanical-turk-now.html"},{"key":"3_CR25","doi-asserted-by":"crossref","unstructured":"Irani, L.C., Silberman, M.S.: Turkopticon: interrupting worker invisibility in Amazon mechanical turk. In: Proceedings of CHI 2013 (CHI 2013), pp. 611\u2013620. ACM (2013)","DOI":"10.1145\/2470654.2470742"},{"key":"3_CR26","unstructured":"Kaufmann, N., Schulze, T., Veit, D.: More than fun and money. Worker motivation in crowdsourcing: a study on mechanical turk. In: Proceedings of the Seventeenth Americas Conference on Information Systems, pp. 1\u201311 (2011)"},{"key":"3_CR27","doi-asserted-by":"crossref","unstructured":"Kazai, G., Kamps, J., Milic-Frayling, N.: The face of quality in crowdsourcing relevance labels: demographics, personality and labeling accuracy. In: Proceedings of the 21st ACM International Conference on Information and Knowledge Management, pp. 2583\u20132586. ACM (2012)","DOI":"10.1145\/2396761.2398697"},{"key":"3_CR28","doi-asserted-by":"crossref","unstructured":"Kittur, A., Nickerson, J.V., Bernstein, M., Gerber, E., Shaw, A., Zimmerman, J., Lease, M., Horton, J.: The future of crowd work. In: Proceedings of the 2013 Conference on Computer Supported Cooperative Work, p. 1301. ACM Press (2013)","DOI":"10.1145\/2441776.2441923"},{"key":"3_CR29","doi-asserted-by":"crossref","unstructured":"Kuek, S.C., Paradi-Guilford, C., Fayomi, T., Imaizumi, S., Ipeirotis, P., Pina, P., Singh, M.: The global opportunity in online outsourcing. Technical report, The World Bank (2015)","DOI":"10.1596\/22284"},{"key":"3_CR30","doi-asserted-by":"crossref","unstructured":"Marshall, C.C., Shipman, F.M.: Experiences surveying the crowd: reflections on methods, participation, and reliability. In: Proceedings of the 5th Annual ACM Web Science Conference, pp. 234\u2013243. ACM (2013)","DOI":"10.1145\/2464464.2464485"},{"key":"3_CR31","doi-asserted-by":"crossref","unstructured":"Martin, D., Hanrahan, B.V., O\u2019Neill, J., Gupta, N.: Being a turker. In: Proceedings of the 17th ACM Conference on Computer Supported Cooperative Work & Social Computing, pp. 224\u2013235. ACM (2014)","DOI":"10.1145\/2531602.2531663"},{"issue":"1","key":"3_CR32","doi-asserted-by":"crossref","first-page":"39","DOI":"10.1007\/s10606-015-9241-6","volume":"25","author":"D Martin","year":"2016","unstructured":"Martin, D., O\u2019Neill, J., Gupta, N., Hanrahan, B.V.: Turking in a global labour market. Comput. Support. Coop. Work (CSCW) 25(1), 39\u201377 (2016)","journal-title":"Comput. Support. Coop. Work (CSCW)"},{"issue":"1","key":"3_CR33","doi-asserted-by":"crossref","first-page":"1","DOI":"10.3758\/s13428-011-0124-6","volume":"44","author":"W Mason","year":"2012","unstructured":"Mason, W., Suri, S.: Conducting behavioral research on Amazons mechanical turk. Behav. Res. Methods 44(1), 1\u201323 (2012)","journal-title":"Behav. Res. Methods"},{"key":"3_CR34","unstructured":"Morozov, E.: To Save Everything, Click Here: The Folly of Technological Solutionism. PublicAffairs (2013)"},{"key":"3_CR35","unstructured":"Naderi, B., Polzehl, T., Beyer, A., Pilz, t., M\u00f6ller, S.: Crowdee: mobile crowdsourcing micro-task platform - for celebrating the diversity of languages. In: Proceedings of the 15th Annual Conference of the International Speech Communication Association (Interspeech 2014). IEEE, September 2014"},{"key":"3_CR36","doi-asserted-by":"crossref","unstructured":"Naderi, B., Wechsung, I., M\u00f6ller, S.: Effect of being observed on the reliability of responses in crowdsourcing micro-task platforms. In: 2015 Seventh International Workshop on Quality of Multimedia Experience (QoMEX), pp. 1\u20132. IEEE (2015)","DOI":"10.1109\/QoMEX.2015.7148091"},{"key":"3_CR37","unstructured":"Naderi, B., Wechsung, I., M\u00f6ller, S.: Crowdsourcing work motivation scale: development and validation for crowdsourcing micro-task platforms. In prepration (2016)"},{"key":"3_CR38","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"429","DOI":"10.1007\/978-3-642-40498-6_33","volume-title":"Human-Computer Interaction \u2013 INTERACT 2013","author":"J O\u2019Neill","year":"2013","unstructured":"O\u2019Neill, J., Martin, D.: Relationship-based business process crowdsourcing? In: Kotz\u00e9, P., Marsden, G., Lindgaard, G., Wesson, J., Winckler, M. (eds.) INTERACT 2013. LNCS, vol. 8120, pp. 429\u2013446. Springer, Heidelberg (2013). doi: 10.1007\/978-3-642-40498-6_33"},{"issue":"3","key":"3_CR39","doi-asserted-by":"crossref","first-page":"184","DOI":"10.1177\/0963721414531598","volume":"23","author":"G Paolacci","year":"2014","unstructured":"Paolacci, G., Chandler, J.: Inside the Turk understanding mechanical turk as a participant pool. Curr. Dir. Psychol. Sci. 23(3), 184\u2013188 (2014)","journal-title":"Curr. Dir. Psychol. Sci."},{"key":"3_CR40","doi-asserted-by":"crossref","unstructured":"Peer, E., Samat, S., Brandimarte, L., Acquisti, A.: Beyond the turk: an empirical comparison of alternative platforms for crowdsourcing online behavioral research (2015). http:\/\/dx.doi.org\/10.2139\/ssrn.2594183","DOI":"10.2139\/ssrn.2594183"},{"key":"3_CR41","doi-asserted-by":"crossref","unstructured":"Ross, J., Irani, L., Silberman, M., Zaldivar, A., Tomlinson, B.: Who are the crowdworkers? Shifting demographics in mechanical turk. In: CHI 2010 Extended Abstracts on Human Factors in Computing Systems, pp. 2863\u20132872. ACM (2010)","DOI":"10.1145\/1753846.1753873"},{"issue":"1","key":"3_CR42","doi-asserted-by":"crossref","first-page":"68","DOI":"10.1037\/0003-066X.55.1.68","volume":"55","author":"RM Ryan","year":"2000","unstructured":"Ryan, R.M., Deci, E.L.: Self-determination theory and the facilitation of intrinsic motivation, social development, and well-being. Am. Psychol. 55(1), 68 (2000)","journal-title":"Am. Psychol."},{"key":"3_CR43","doi-asserted-by":"crossref","unstructured":"Salehi, N., Irani, L.C., Bernstein, M.S., Alkhatib, A., Ogbe, E., Milland, K., et al.: We are dynamo: overcoming stalling and friction in collective action for crowd workers. In: Proceedings of the 33rd Annual ACM Conference on Human Factors in Computing Systems, pp. 1621\u20131630. ACM (2015)","DOI":"10.1145\/2702123.2702508"},{"issue":"2","key":"3_CR44","doi-asserted-by":"crossref","first-page":"39","DOI":"10.1145\/1869086.1869100","volume":"17","author":"M Silberman","year":"2010","unstructured":"Silberman, M., Irani, L., Ross, J.: Ethics and tactics of professional crowdwork. XRDS Crossroads ACM Mag. Stud. 17(2), 39\u201343 (2010)","journal-title":"XRDS Crossroads ACM Mag. Stud."},{"key":"3_CR45","unstructured":"Silberman, M.S.: What\u2019s fair? Rational action and its residuals in an electronic market. Unpublished manuscript (2010). http:\/\/www.scribd.com\/doc\/86592724\/Whats-Fair"},{"key":"3_CR46","unstructured":"Silberman, S., Milland, K., LaPlante, R., Ross, J., Irani, L.: Stop citing Ross et al. 2010, Who are the crowdworkers? (2015). https:\/\/medium.com\/@silberman\/stop-citing-ross-et-al-2010-who-are-the-crowdworkers-b3b9b1e8d300"}],"container-title":["Lecture Notes in Computer Science","Evaluation in the Crowd. Crowdsourcing and Human-Centered Experiments"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-319-66435-4_3","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,25]],"date-time":"2025-06-25T22:31:41Z","timestamp":1750890701000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-319-66435-4_3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017]]},"ISBN":["9783319664347","9783319664354"],"references-count":46,"URL":"https:\/\/doi.org\/10.1007\/978-3-319-66435-4_3","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2017]]}}}