{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,20]],"date-time":"2025-11-20T12:50:33Z","timestamp":1763643033631,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":37,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,11,19]],"date-time":"2020-11-19T00:00:00Z","timestamp":1605744000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020,11,19]]},"DOI":"10.1145\/3428029.3428062","type":"proceedings-article","created":{"date-parts":[[2020,11,22]],"date-time":"2020-11-22T20:32:05Z","timestamp":1606077125000},"page":"1-5","source":"Crossref","is-referenced-by-count":7,"title":["Crowdsourcing in Computing Education Research: Case\u00a0Amazon\u00a0MTurk"],"prefix":"10.1145","author":[{"given":"Arto","family":"Hellas","sequence":"first","affiliation":[{"name":"Aalto University, Finland"}]},{"given":"Albina","family":"Zavgorodniaia","sequence":"additional","affiliation":[{"name":"Aalto University, Finland"}]},{"given":"Juha","family":"Sorva","sequence":"additional","affiliation":[{"name":"Aalto University, Finland"}]}],"member":"320","published-online":{"date-parts":[[2020,11,22]]},"reference":[{"volume-title":"Mechanical Turk. In Meeting of the Midwest Political Science Association.","year":"2019","author":"Ahler J","key":"e_1_3_2_1_1_1"},{"volume-title":"Worker identity and Requester tools to manage task quality.","year":"2020","author":"Mechanical\u00a0Turk Blog Amazon","key":"e_1_3_2_1_2_1"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.2308\/bria-52340"},{"key":"e_1_3_2_1_4_1","unstructured":"Casey Casalnuovo Prem Devanbu and Emily Morgan. [n.d.]. Does Surprisal Predict Code Comprehension Difficulty?([n.\u00a0d.]).  Casey Casalnuovo Prem Devanbu and Emily Morgan. [n.d.]. Does Surprisal Predict Code Comprehension Difficulty?([n.\u00a0d.])."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2017.8115653"},{"volume-title":"Proceedings of the 45th ACM technical symposium on Computer science education. 653\u2013658","year":"2014","author":"Charters Polina","key":"e_1_3_2_1_6_1"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2232817.2232840"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1177\/1948550619875149"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/1404520.1404526"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.5555\/1862712.1862724"},{"volume-title":"Proceedings of the 42nd ACM technical symposium on Computer science education. 471\u2013476","year":"2011","author":"Denny Paul","key":"e_1_3_2_1_11_1"},{"volume-title":"2019 34th IEEE\/ACM Int. Conf. on Automated Software Engineering (ASE). IEEE, 399\u2013410","year":"2019","author":"Endres Madeline","key":"e_1_3_2_1_12_1"},{"key":"e_1_3_2_1_13_1","article-title":"Towards an integrated crowdsourcing definition","volume":"38","author":"Estell\u00e9s-Arolas Enrique","year":"2012","journal-title":"J. of Inf. Science"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2015.7357193"},{"volume-title":"Proceedings of the Thirty-Fifth Annual Conference of the Cognitive Science Society","year":"2013","author":"Hansen ME","key":"e_1_3_2_1_15_1"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"David Hauser Gabriele Paolacci and Jesse\u00a0J Chandler. 2018. Common concerns with MTurk as a participant pool: Evidence and solutions. (2018).  David Hauser Gabriele Paolacci and Jesse\u00a0J Chandler. 2018. Common concerns with MTurk as a participant pool: Evidence and solutions. (2018).","DOI":"10.31234\/osf.io\/uq45c"},{"volume-title":"The rise of crowdsourcing. Wired magazine 14, 6","year":"2006","author":"Howe Jeff","key":"e_1_3_2_1_17_1"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.20982\/tqmp.16.3.p216"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Ryan Kennedy Scott Clifford Tyler Burleigh Philip\u00a0D Waggoner Ryan Jewell and Nicholas\u00a0JG Winter. 2018. The shape of and solutions to the MTurk quality crisis. Political Science Research and Methods(2018) 1\u201316.  Ryan Kennedy Scott Clifford Tyler Burleigh Philip\u00a0D Waggoner Ryan Jewell and Nicholas\u00a0JG Winter. 2018. The shape of and solutions to the MTurk quality crisis. Political Science Research and Methods(2018) 1\u201316.","DOI":"10.2139\/ssrn.3272468"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2016911.2016934"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2012.6344507"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2787622.2787709"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2493394.2493410"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341525.3387385"},{"volume-title":"The Twenty-Ninth International Flairs Conference.","year":"2016","author":"Luger KK","key":"e_1_3_2_1_25_1"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3291279.3339420"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3304221.3319759"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3105726.3106191"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3197091.3197117"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3279720.3279741"},{"key":"e_1_3_2_1_32_1","unstructured":"Alana Semuels. 2018. The internet is enabling a new kind of poorly paid hell. The Atlantic 23(2018).  Alana Semuels. 2018. The internet is enabling a new kind of poorly paid hell. The Atlantic 23(2018)."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2019.8818950"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3393673.3276947"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3133850.3133863"},{"volume-title":"Proc. of the Sixth (2019)","year":"2019","author":"Wang Xu","key":"e_1_3_2_1_36_1"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3416465.3416468"},{"volume-title":"Proceedings of the 20th Koli Calling International Conference on Computing Education Research(Koli Calling \u201920)","year":"2020","author":"Zavgorodniaia Albina","key":"e_1_3_2_1_38_1"}],"event":{"name":"Koli Calling '20: 20th Koli Calling International Conference on Computing Education Research","acronym":"Koli Calling '20","location":"Koli Finland"},"container-title":["Koli Calling '20: Proceedings of the 20th Koli Calling International Conference on Computing Education Research"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3428029.3428062","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3428029.3428062","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T21:24:23Z","timestamp":1750195463000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3428029.3428062"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,11,19]]},"references-count":37,"alternative-id":["10.1145\/3428029.3428062","10.1145\/3428029"],"URL":"https:\/\/doi.org\/10.1145\/3428029.3428062","relation":{},"subject":[],"published":{"date-parts":[[2020,11,19]]}}}