{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,24]],"date-time":"2026-06-24T07:42:25Z","timestamp":1782286945616,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":69,"publisher":"ACM","license":[{"start":{"date-parts":[[2018,4,21]],"date-time":"2018-04-21T00:00:00Z","timestamp":1524268800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Economic and Social Research Council","award":["Grant ES\/N017099\/1"],"award-info":[{"award-number":["Grant ES\/N017099\/1"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2018,4,21]]},"DOI":"10.1145\/3173574.3174023","type":"proceedings-article","created":{"date-parts":[[2018,4,20]],"date-time":"2018-04-20T15:38:35Z","timestamp":1524238715000},"page":"1-14","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":309,"title":["A Data-Driven Analysis of Workers' Earnings on Amazon Mechanical Turk"],"prefix":"10.1145","author":[{"given":"Kotaro","family":"Hara","sequence":"first","affiliation":[{"name":"Singapore Management University & Carnegie Mellon University, Singapore, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Abigail","family":"Adams","sequence":"additional","affiliation":[{"name":"University of Oxford, Oxford, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kristy","family":"Milland","sequence":"additional","affiliation":[{"name":"McMaster University & TurkerNation.com, Toronto, ON, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Saiph","family":"Savage","sequence":"additional","affiliation":[{"name":"West Virginia University, Morgantown, WV, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chris","family":"Callison-Burch","sequence":"additional","affiliation":[{"name":"University of Pennsylvania, Philadelphia, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jeffrey P.","family":"Bigham","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2018,4,21]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"crossref","unstructured":"Abigail Adams and Janine Berg. 2017. When Home Affects Pay: An Analysis of the Gender Pay Gap among Crowdworkers.","DOI":"10.2139\/ssrn.3048711"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025974"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1979742.1979606"},{"key":"e_1_3_2_2_4_1","unstructured":"Janine Berg. 2016. Income security in the ondemand economy: findings and policy lessons from a survey of crowdworkers."},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1866029.1866078"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","unstructured":"Jeffrey P Bigham Chandrika Jayant Hanjie Ji Greg Little Andrew Miller Robert C. Miller Robin Miller Aubrey Tatarowicz Brandyn White Samuel White and Tom Yeh. 2010. VizWiz: Nearly Real-Time Answers to Visual Questions. Uist: 333--342. 10.1145\/1866029.1866080","DOI":"10.1145\/1866029.1866080"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.5555\/944919.944937"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"crossref","unstructured":"John Bohannon. 2011. Social Science for Pennies. Science 334 6054: 307 LP-307. Retrieved from http:\/\/science.sciencemag.org\/content\/334\/6054\/307 .abstract","DOI":"10.1126\/science.334.6054.307"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/1458082.1458105"},{"key":"e_1_3_2_2_10_1","volume-title":"Americans with Disabilities","author":"Brault Matthew W.","year":"2010","unstructured":"Matthew W. Brault. 2012. Americans with Disabilities: 2010, Household Economics Studies. Retrieved from http:\/\/www.census.gov\/prod\/2012pubs\/p70--131.pdf"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.chb.2015.08.031"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.chb.2015.08.031"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858198"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v2i1.13198"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.5555\/1866696.1866697"},{"key":"e_1_3_2_2_16_1","volume-title":"International Conference on Machine Learning (ICML).","author":"Chuang Jason","year":"2013","unstructured":"Jason Chuang, Sonal Gupta, Christopher D Manning, and Jeffrey Heer. 2013. Topic Model Diagnostics: Assessing Domain Relevance via Topical Alignment. In International Conference on Machine Learning (ICML). Retrieved from http:\/\/vis.stanford.edu\/papers\/topic-modeldiagnostics"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2207738"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"crossref","unstructured":"Jason Chuang Margaret E Roberts Brandon M Stewart Rebecca Weiss Dustin Tingley Justin Grimmer and Jeffrey Heer. 2015. TopicCheck: Interactive Alignment for Assessing Topic Model Stability. In HLT-NAACL 175--184.","DOI":"10.3115\/v1\/N15-1018"},{"key":"e_1_3_2_2_19_1","unstructured":"U.S. District Court. 2015. Christopher Otey et al. v. CrowdFlower INC. et al."},{"key":"e_1_3_2_2_20_1","unstructured":"Carroll Croarkin Paul Tobias and Chelli Zey. 2002. Engineering statistics handbook. NIST iTL."},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2736277.2741685"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1019637632584"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1057\/ejis.2010.61"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.5555\/2908698.2908703"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2631775.2631819"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2660398.2660403"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1016\/0030-5073(76)90016-7"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.5555\/1613715.1613763"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702279"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132525.3132544"},{"key":"e_1_3_2_2_31_1","volume-title":"Krueger","author":"Harris Seth D.","year":"2015","unstructured":"Seth D. Harris and Alan B. Krueger. 2015. A Proposal for Modernizing Labor Laws for TwentyFirst-Century Work: The \"Independent Worker.\""},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.5555\/540298"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.3233\/JVR-2010-0502"},{"key":"e_1_3_2_2_34_1","volume-title":"Research in the crowdsourcing age, a case study","author":"Hitlin Paul","unstructured":"Paul Hitlin. 2016. Research in the crowdsourcing age, a case study. Pew Research Center."},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/1807342.1807376"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858327"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/1869086.1869094"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/2470654.2470742"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","unstructured":"Lilly C Irani and M Six Silberman. 2016. Stories We Tell About Labor: Turkopticon and the Trouble with \"Design.\" In Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems (CHI '16) 4573--4586. 10.1145\/2858036.2858592","DOI":"10.1145\/2858036.2858592"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.14778\/3067421.3067431"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.5555\/2517747"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"crossref","unstructured":"Miranda Katz. 2017. Amazon Mechanical Turk Workers Have Had Enough. Wired.","DOI":"10.1002\/9781405165518.wbeos0994"},{"key":"e_1_3_2_2_43_1","volume-title":"Proc. of AMCIS","author":"Kaufmann Nicolas","year":"2011","unstructured":"Nicolas Kaufmann, Thimo Schulze, and Daniel Veit. 2011. More than fun and money. worker motivation in crowdsourcing--a study on mechanical turk. In Proc. of AMCIS 2011."},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/2441776.2441923"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","unstructured":"Adriana Kovashka Olga Russakovsky Li Fei-Fei and Kristen Grauman. 2016. Crowdsourcing in Computer Vision. Foundations and Trends\u00ae in Computer Graphics and Vision 10 3: 177--243.","DOI":"10.5555\/3125988"},{"key":"e_1_3_2_2_46_1","unstructured":"Siou Chew Kuek Cecilia Paradi-Guilford Toks Fayomi Saori Imaizumi Panos Ipeirotis Patricia Pina Manpreet Singh and others. 2015. The global opportunity in online outsourcing."},{"key":"e_1_3_2_2_47_1","volume-title":"Martine Humblet, Christina Behrendt, Susan Hayter, Christopher Land-Kazlauskas, Angelika Muller, Niall O'Higgins, Friederike Eberlein, Simone Marino, Bao Chau Le, and Calum Carson.","author":"Marcadent Philippe","year":"2016","unstructured":"Philippe Marcadent, Janine Berg, Mariya Aleksynska, Valerio De Stefano, Martine Humblet, Christina Behrendt, Susan Hayter, Christopher Land-Kazlauskas, Angelika Muller, Niall O'Higgins, Friederike Eberlein, Simone Marino, Bao Chau Le, and Calum Carson. 2016. NonStandard Employment Around The World."},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","unstructured":"Adam Marcus and Aditya Parameswaran. 2015. Crowdsourced Data Management: Industry and Academic Perspectives. Foundations and Trends\u00ae in Databases 6 1--2: 1--161. 10.1561\/1900000044","DOI":"10.1561\/1900000044"},{"key":"e_1_3_2_2_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/2531602.2531663"},{"key":"e_1_3_2_2_50_1","unstructured":"Alexandre Mas and Amanda Pallais. 2016. Valuing alternative work arrangements."},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","unstructured":"Winter Mason and Duncan J Watts. 2009. Financial Incentives and the \"Performance of Crowds.\" In Proceedings of the ACM SIGKDD Workshop on Human Computation (HCOMP '09) 77--85. 10.1145\/1600150.1600175","DOI":"10.1145\/1600150.1600175"},{"key":"e_1_3_2_2_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858539"},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2013.200"},{"key":"e_1_3_2_2_54_1","doi-asserted-by":"publisher","DOI":"10.5555\/1953048.2078195"},{"key":"e_1_3_2_2_55_1","volume-title":"ALTO: Active Learning with Topic Overviews for Speeding Label Induction and Document Labeling. In ACL (1).","author":"Poursabzi-Sangdeh Forough","year":"2016","unstructured":"Forough Poursabzi-Sangdeh, Jordan L BoydGraber, Leah Findlater, and Kevin D Seppi. 2016. ALTO: Active Learning with Topic Overviews for Speeding Label Induction and Document Labeling. In ACL (1)."},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979148"},{"key":"e_1_3_2_2_57_1","volume-title":"Proceedings of the first instructional conference on machine learning, 133--142","author":"Juan","unstructured":"Juan Ramos and others. 2003. Using tf-idf to determine word relevance in document queries. In Proceedings of the first instructional conference on machine learning, 133--142."},{"key":"e_1_3_2_2_58_1","unstructured":"Reddit. \/r\/mturk: Crowd-Workers browser extension discussion. Retrieved from https:\/\/www.reddit.com\/r\/mturk\/comments\/35uujl\/c rowdworkers_browser_extension_discussion"},{"key":"e_1_3_2_2_59_1","volume-title":"Proceedings of the LREC 2010 Workshop on New Challenges for NLP Frameworks, 45--50","author":"Rehurek Radim","year":"2010","unstructured":"Radim Rehurek and Petr Sojka. 2010. Software Framework for Topic Modelling with Large Corpora. In Proceedings of the LREC 2010 Workshop on New Challenges for NLP Frameworks, 45--50."},{"key":"e_1_3_2_2_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702508"},{"key":"e_1_3_2_2_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/1869086.1869100"},{"key":"e_1_3_2_2_62_1","doi-asserted-by":"publisher","unstructured":"Yaron Singer and Manas Mittal. 2011. Pricing Tasks in Online Labor Markets. In Human Computation.","DOI":"10.5555\/2908698.2908708"},{"key":"e_1_3_2_2_63_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1008651105359"},{"key":"e_1_3_2_2_64_1","doi-asserted-by":"publisher","DOI":"10.1038\/nmeth.1619"},{"key":"e_1_3_2_2_65_1","unstructured":"David Weil. 2015. The Application of the Fair Labor Standards Act's \"Suffer or Permit\" Standard in the Identification of Employees Who Are Misclassified as Independent Contractors."},{"key":"e_1_3_2_2_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/2675133.2675158"},{"key":"e_1_3_2_2_67_1","volume-title":"Retrieved","author":"Turk Participation Agreement Amazon Mechanical","year":"2014","unstructured":"Amazon Mechanical Turk Participation Agreement (2014). Retrieved July 1, 2017 from https:\/\/www.mturk.com\/mturk\/conditionsofuse"},{"key":"e_1_3_2_2_68_1","unstructured":"Turker Nation. Retrieved September 1 2017 from http:\/\/www.turkernation.com\/"},{"key":"e_1_3_2_2_69_1","unstructured":"The Evolving Ecosystem of Crowd Workers. Retrieved September 1 2017 from http:\/\/www.ifis.cs.tubs.de\/project\/crowdsourcing\/ecosystem"}],"event":{"name":"CHI '18: CHI Conference on Human Factors in Computing Systems","location":"Montreal QC Canada","acronym":"CHI '18","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2018 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3173574.3174023","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3173574.3174023","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T01:08:37Z","timestamp":1750208917000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3173574.3174023"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,4,21]]},"references-count":69,"alternative-id":["10.1145\/3173574.3174023","10.1145\/3173574"],"URL":"https:\/\/doi.org\/10.1145\/3173574.3174023","relation":{},"subject":[],"published":{"date-parts":[[2018,4,21]]},"assertion":[{"value":"2018-04-21","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}