{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,26]],"date-time":"2025-12-26T07:14:49Z","timestamp":1766733289420,"version":"3.37.3"},"reference-count":71,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/501100004022","name":"Jenny and Antti Wihuri Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100004022","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004757","name":"Ulla Tuomisen S\u00e4\u00e4ti\u00f6","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100004757","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2023]]},"DOI":"10.1109\/access.2023.3253642","type":"journal-article","created":{"date-parts":[[2023,3,6]],"date-time":"2023-03-06T18:44:00Z","timestamp":1678128240000},"page":"22982-22992","source":"Crossref","is-referenced-by-count":3,"title":["Lessons Learned From Four Computing Education Crowdsourcing Systems"],"prefix":"10.1109","volume":"11","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5249-5162","authenticated-orcid":false,"given":"Nea","family":"Pirttinen","sequence":"first","affiliation":[{"name":"Department of Computer Science, University of Helsinki, Helsinki, Finland"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5150-9806","authenticated-orcid":false,"given":"Paul","family":"Denny","sequence":"additional","affiliation":[{"name":"School of Computer Science, The University of Auckland, Auckland, New Zealand"}]},{"given":"Arto","family":"Hellas","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Aalto University, Espoo, Finland"}]},{"given":"Juho","family":"Leinonen","sequence":"additional","affiliation":[{"name":"School of Computer Science, The University of Auckland, Auckland, New Zealand"}]}],"member":"263","reference":[{"key":"ref1","first-page":"69","article-title":"The PeerWise system of student contributed assessment questions","volume-title":"Proc. 10th Conf. Australas. Comput. Educ.","author":"Denny"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1145\/3197091.3197117"},{"volume-title":"Learnersourcing: Improving learning with collective learner activity","year":"2015","author":"Kim","key":"ref3"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.18608\/jla.2019.63.12"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1145\/1953163.1953299"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1145\/3341525.3387385"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1007\/1-4020-3669-8_3"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1145\/1140124.1140145"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1145\/1473195.1473242"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/FIE.2013.6685132"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1080\/08993408.2010.486275"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1145\/3430665.3456338"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1145\/3403935"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1207\/s15516709cog1302_1"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1207\/s15327809jls0201_1"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1037\/0278-7393.15.4.669"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.3758\/BF03196872"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.2307\/1423462"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1080\/01449298208914461"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1016\/j.jecp.2008.03.001"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1177\/0098628319834174"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1111\/j.1467-9280.2006.01693.x"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.3758\/BF03202713"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1016\/S0022-5371(78)90393-6"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1590\/1982-43272356201314"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.3758\/s13415-016-0407-6"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.5214\/ans.0972.7531.200408"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1038\/nrn.2015.18"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1177\/1948550619875149"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1017\/psrm.2020.6"},{"article-title":"Effects of visualization on algorithm comprehension","year":"2015","author":"Mulvey","key":"ref31"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1145\/2016911.2016934"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/3428029.3428062"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2012.6344507"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1145\/3133850.3133863"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1145\/2787622.2787709"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376283"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1145\/2676723.2677253"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173885"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1111\/bjet.12686"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1145\/3300115.3312507"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1145\/2462476.2462501"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/TLT.2021.3058644"},{"key":"ref44","first-page":"137","article-title":"The quality of a PeerWise MCQ repository","volume-title":"Proc. 12th Australas. Conf. Comput. Educ.","volume":"103","author":"Purchase"},{"key":"ref45","first-page":"55","article-title":"Quality of student contributed questions using PeerWise","volume-title":"Proc. 11th Australas. Conf. Comput. Educ.","volume":"95","author":"Denny"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1145\/3279720.3279741"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1145\/1595496.1562888"},{"key":"ref48","first-page":"1","article-title":"Exploring the complexity of crowdsourced programming assignments","volume-title":"Proc. 7th SPLICE Workshop (SIGCSE)","author":"Pirttinen"},{"article-title":"On the quality of crowdsourced programming assignments","year":"2020","author":"Pirttinen","key":"ref49"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1007\/BF02209024"},{"issue":"1","key":"ref51","first-page":"5","article-title":"Technology enhanced peer learning with PeerWise: Experiences and perceptions from a developing country","volume":"4","author":"Singh","year":"2014","journal-title":"Caribbean Teach. Scholar"},{"volume-title":"Teaching for Quality Learning at University","year":"2011","author":"Biggs","key":"ref52"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1145\/1999747.1999807"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.11120\/ital.2012.11010027"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1145\/3430895.3460142"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1145\/2554850.2554956"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1109\/HICSS.2014.377"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1007\/s10639-020-10228-x"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1177\/1046878114563660"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1145\/3059009.3059033"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2020.3024102"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1145\/3408877.3432430"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1145\/2828959.2828967"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1145\/3300115.3309516"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1080\/08993408.2012.727712"},{"key":"ref66","first-page":"24","article-title":"Integrating CrowdSorcerer: Lessons learned","volume-title":"Proc. Workshop Comput. Sci. Educ. Infrastruct.","author":"Pirttinen"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v3i1.13229"},{"key":"ref68","first-page":"100","article-title":"A comparison of anonymous versus identifiable e-peer review on college student writing performance and the extent of critical feedback","volume":"6","author":"Lu","year":"2007","journal-title":"J. Interact. Online Learn."},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.2190\/EC.43.1.f"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1145\/3502718.3524762"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1109\/LaTiCE.2013.34"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6287639\/10005208\/10061397.pdf?arnumber=10061397","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,2,13]],"date-time":"2024-02-13T09:39:33Z","timestamp":1707817173000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10061397\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"references-count":71,"URL":"https:\/\/doi.org\/10.1109\/access.2023.3253642","relation":{},"ISSN":["2169-3536"],"issn-type":[{"type":"electronic","value":"2169-3536"}],"subject":[],"published":{"date-parts":[[2023]]}}}