{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,23]],"date-time":"2026-06-23T03:04:53Z","timestamp":1782183893049,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":80,"publisher":"ACM","license":[{"start":{"date-parts":[[2010,10,28]],"date-time":"2010-10-28T00:00:00Z","timestamp":1288224000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2010,10,28]]},"DOI":"10.1145\/1930464.1930480","type":"proceedings-article","created":{"date-parts":[[2011,1,25]],"date-time":"2011-01-25T13:34:48Z","timestamp":1295962488000},"page":"86-93","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":363,"title":["Review of recent systems for automatic assessment of programming assignments"],"prefix":"10.1145","author":[{"given":"Petri","family":"Ihantola","sequence":"first","affiliation":[{"name":"Aalto University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tuukka","family":"Ahoniemi","sequence":"additional","affiliation":[{"name":"Digia Plc, Finland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ville","family":"Karavirta","sequence":"additional","affiliation":[{"name":"Aalto University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Otto","family":"Sepp\u00e4l\u00e4","sequence":"additional","affiliation":[{"name":"Aalto University"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2010,10,28]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1080\/08993400500150747"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1268784.1268813"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1384271.1384330"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1140124.1140150"},{"key":"e_1_3_2_1_5_1","volume-title":"Thid edition","author":"Barnes D. J.","year":"2006","unstructured":"D. J. Barnes and M. K\u00f6lling . Objects First with Java - A Practical Introduction using BlueJ , Thid edition . Prentice Hall \/ Pearson Education , 2006 . D. J. Barnes and M. K\u00f6lling. Objects First with Java - A Practical Introduction using BlueJ, Thid edition. Prentice Hall \/ Pearson Education, 2006."},{"key":"e_1_3_2_1_6_1","volume-title":"Teaching for Quality Learning at University: What the Student Does","author":"Biggs J.","year":"2007","unstructured":"J. Biggs and C. Tang . Teaching for Quality Learning at University: What the Student Does ( 3 rd Edition). Open University Press , 2007 . J. Biggs and C. Tang. Teaching for Quality Learning at University: What the Student Does (3rd Edition). Open University Press, 2007.","edition":"3"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2006.07.009"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1163405.1163411"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/960492.960539"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/1140124.1140159"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/1734263.1734295"},{"key":"e_1_3_2_1_12_1","first-page":"107","volume-title":"CCSC '00: Proceedings of the fifth annual CCSC northeastern Conf. on The journal of computing in small colleges","author":"Cooper S.","year":"2000","unstructured":"S. Cooper , W. Dann , and R. Pausch . Alice: a 3-d tool for introductory programming concepts . In CCSC '00: Proceedings of the fifth annual CCSC northeastern Conf. on The journal of computing in small colleges , pages 107 -- 116 , USA, 2000 . Consortium for Computing Sciences in Colleges. S. Cooper, W. Dann, and R. Pausch. Alice: a 3-d tool for introductory programming concepts. In CCSC '00: Proceedings of the fifth annual CCSC northeastern Conf. on The journal of computing in small colleges, pages 107--116, USA, 2000. Consortium for Computing Sciences in Colleges."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1315803.1315821"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/1163405.1163409"},{"issue":"3","key":"e_1_3_2_1_15_1","first-page":"44","article-title":"Experiences using test-driven development with an automated grader","volume":"22","author":"Edwards S. H.","year":"2007","unstructured":"S. H. Edwards and M. A. P\u00e9rez-Qui\u00f1ones . Experiences using test-driven development with an automated grader . J. Comput. Small Coll. , 22 ( 3 ): 44 -- 50 , 2007 . S. H. Edwards and M. A. P\u00e9rez-Qui\u00f1ones. Experiences using test-driven development with an automated grader. J. Comput. Small Coll., 22(3):44--50, 2007.","journal-title":"J. Comput. Small Coll."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/1384271.1384371"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.5555\/1149707.1149713"},{"key":"e_1_3_2_1_18_1","volume-title":"Evolution and Perspectives","author":"Fori\u0161ek M.","year":"2006","unstructured":"M. Fori\u0161ek . Security of programming contest systems. In Informatics in Secondary Schools , Evolution and Perspectives , Vilnius, Lithuania , 2006 . M. Fori\u0161ek. Security of programming contest systems. In Informatics in Secondary Schools, Evolution and Perspectives, Vilnius, Lithuania, 2006."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/1352135.1352163"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/1595496.1562897"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.5555\/1323159.1323237"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/1294325.1294327"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/1597849.1384329"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/1140124.1140139"},{"key":"e_1_3_2_1_25_1","first-page":"1","volume-title":"Computer Science Education in the 21st Century","author":"Greening T.","year":"1999","unstructured":"T. Greening . Computer Science Educational Futures : The Nature of 2020\" Foresight . In Computer Science Education in the 21st Century , pages 1 -- 6 . Springer Verlag , 1999 . T. Greening. Computer Science Educational Futures: The Nature of 2020\" Foresight. In Computer Science Education in the 21st Century, pages 1--6. Springer Verlag, 1999."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/1140124.1140130"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.compedu.2009.09.006"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/1168812.1168827"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/1268784.1268805"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1026364126982"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jvlc.2006.09.001"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.intcom.2008.07.002"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.5555\/1322395.1322402"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/268084.268210"},{"key":"e_1_3_2_1_35_1","volume-title":"Proceedings of the Fourth Finnish \/ Baltic Sea Conf. on Computer Science Education","volume":"04","author":"Janhunen T.","year":"2004","unstructured":"T. Janhunen , T. Jussila , M. J\u00e4rvisalo , and E. Oikarinen . Teaching Smullyan's analytic tableaux in a scalable learning environment . In Proceedings of the Fourth Finnish \/ Baltic Sea Conf. on Computer Science Education , volume TKO-42\/ 04 of Research Report Series of Laboratory of Information Processing Science, Helsinki University of Technology, pages 85--94. Otamedia , December 2004 . T. Janhunen, T. Jussila, M. J\u00e4rvisalo, and E. Oikarinen. Teaching Smullyan's analytic tableaux in a scalable learning environment. In Proceedings of the Fourth Finnish \/ Baltic Sea Conf. on Computer Science Education, volume TKO-42\/04 of Research Report Series of Laboratory of Information Processing Science, Helsinki University of Technology, pages 85--94. Otamedia, December 2004."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/FIE.2008.4720591"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/1163405.1163407"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/EmbeddedCom-ScalCom.2009.105"},{"key":"e_1_3_2_1_39_1","first-page":"38","article-title":"Infrastructure for contest task development","volume":"3","author":"Kolstad R.","year":"2009","unstructured":"R. Kolstad . Infrastructure for contest task development . Olympiads in Informatics , 3 : 38 -- 59 , 2009 . R. Kolstad. Infrastructure for contest task development. Olympiads in Informatics, 3:38--59, 2009.","journal-title":"Olympiads in Informatics"},{"key":"e_1_3_2_1_40_1","first-page":"48","volume-title":"Proceedings of the Third Annual Baltic Conf. on Computer Science Education","author":"Korhonen A.","year":"2003","unstructured":"A. Korhonen , L. Malmi , and P. Silvasti . TRAKLA2: a framework for automatically assessed visual algorithm simulation exercises . In Proceedings of the Third Annual Baltic Conf. on Computer Science Education , pages 48 -- 56 , Joensuu, Finland , 2003 . A. Korhonen, L. Malmi, and P. Silvasti. TRAKLA2: a framework for automatically assessed visual algorithm simulation exercises. In Proceedings of the Third Annual Baltic Conf. on Computer Science Education, pages 48--56, Joensuu, Finland, 2003."},{"key":"e_1_3_2_1_41_1","first-page":"41","volume-title":"Koli Calling 2005 -- Fifth Koli Calling Conf. on Computer Science Education","author":"Lehtonen T.","year":"2005","unstructured":"T. Lehtonen . Javala -- addictive e-learning of the java programming language. In Koli Calling 2005 -- Fifth Koli Calling Conf. on Computer Science Education , pages 41 -- 48 , 2005 . T. Lehtonen. Javala -- addictive e-learning of the java programming language. In Koli Calling 2005 -- Fifth Koli Calling Conf. on Computer Science Education, pages 41--48, 2005."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/CISE.2009.5365307"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/CSSE.2008.111"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/1163405.1163412"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/1159733.1159778"},{"key":"e_1_3_2_1_46_1","first-page":"124","article-title":"Perspectives on grading systems","volume":"1","author":"Mare\u0161 M.","year":"2007","unstructured":"M. Mare\u0161 . Perspectives on grading systems . Olympiads in Informatics , 1 : 124 -- 130 , 2007 . M. Mare\u0161. Perspectives on grading systems. Olympiads in Informatics, 1:124--130, 2007.","journal-title":"Olympiads in Informatics"},{"key":"e_1_3_2_1_47_1","first-page":"60","article-title":"Moe -- design of a modular grading system","volume":"3","author":"Mare\u0161 M.","year":"2009","unstructured":"M. Mare\u0161 . Moe -- design of a modular grading system . Olympiads in Informatics , 3 : 60 -- 66 , 2009 . M. Mare\u0161. Moe -- design of a modular grading system. Olympiads in Informatics, 3:60--66, 2009.","journal-title":"Olympiads in Informatics"},{"key":"e_1_3_2_1_48_1","first-page":"67","article-title":"Using a linux security module for contest security","volume":"3","author":"Merry B.","year":"2009","unstructured":"B. Merry . Using a linux security module for contest security . Olympiads in Informatics , 3 : 67 -- 73 , 2009 . B. Merry. Using a linux security module for contest security. Olympiads in Informatics, 3:67--73, 2009.","journal-title":"Olympiads in Informatics"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/989863.989928"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.compedu.2009.09.005"},{"issue":"2","key":"e_1_3_2_1_51_1","first-page":"16","article-title":"Providing accurate and timely feedback by automatically grading student programming labs","volume":"23","author":"Nordquist P.","year":"2007","unstructured":"P. Nordquist . Providing accurate and timely feedback by automatically grading student programming labs . J. Comput. Small Coll. , 23 ( 2 ): 16 -- 23 , 2007 . P. Nordquist. Providing accurate and timely feedback by automatically grading student programming labs. J. Comput. Small Coll., 23(2):16--23, 2007.","journal-title":"J. Comput. Small Coll."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/1268784.1268799"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/1140124.1140182"},{"key":"e_1_3_2_1_54_1","volume-title":"Karel the robot","author":"Pattis R. E.","year":"1994","unstructured":"R. E. Pattis , J. Roberts , and M. Stehlik . Karel the robot ( 2 nd ed.): a gentle introduction to the art of programming. John Wiley & amp; Sons, Inc., New York, NY, USA, 1994 . R. E. Pattis, J. Roberts, and M. Stehlik. Karel the robot (2nd ed.): a gentle introduction to the art of programming. John Wiley & Sons, Inc., New York, NY, USA, 1994.","edition":"2"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/1113847.1113893"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/1384271.1384368"},{"key":"e_1_3_2_1_57_1","first-page":"131","article-title":"Competitive learning in informatics: The uva online judge experience","volume":"2","author":"Revilla M. A.","year":"2008","unstructured":"M. A. Revilla , S. Manzoor , and R. Liu . Competitive learning in informatics: The uva online judge experience . Olympiads in Informatics , 2 : 131 -- 148 , 2008 . M. A. Revilla, S. Manzoor, and R. Liu. Competitive learning in informatics: The uva online judge experience. Olympiads in Informatics, 2:131--148, 2008.","journal-title":"Olympiads in Informatics"},{"key":"e_1_3_2_1_58_1","first-page":"139","article-title":"Increasing the appeal of programming contests with tasks involving graphical user interfaces and computer graphics","volume":"1","author":"Ribeiro P.","year":"2007","unstructured":"P. Ribeiro and P. Guerreiro . Increasing the appeal of programming contests with tasks involving graphical user interfaces and computer graphics . Olympiads in Informatics , 1 : 139 -- 164 , 2007 . P. Ribeiro and P. Guerreiro. Increasing the appeal of programming contests with tasks involving graphical user interfaces and computer graphics. Olympiads in Informatics, 1:139--164, 2007.","journal-title":"Olympiads in Informatics"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/1562877.1562929"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1006\/jvlc.2002.0239"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/1597849.1384405"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/1473195.1473239"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/1538234.1538238"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0169-2070(99)00018-7"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/1562877.1562971"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/1352135.1352317"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/1138929.1138951"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/1584322.1584334"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/1562877.1562899"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1109\/ITHET.2006.339754"},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/1562877.1562893"},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1145\/1140124.1140131"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1109\/2.58216"},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.1145\/1456659.1456686"},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.1145\/1593105.1593128"},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1145\/1268784.1268806"},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.1145\/1352135.1352316"},{"key":"e_1_3_2_1_78_1","doi-asserted-by":"publisher","DOI":"10.1145\/971300.971391"},{"key":"e_1_3_2_1_79_1","first-page":"192","article-title":"Programming task packages: Peach exchange format","volume":"2","author":"Verhoeff T.","year":"2008","unstructured":"T. Verhoeff . Programming task packages: Peach exchange format . Olympiads in Informatics , 2 : 192 -- 207 , 2008 . T. Verhoeff. Programming task packages: Peach exchange format. Olympiads in Informatics, 2:192--207, 2008.","journal-title":"Olympiads in Informatics"},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2006.03.001"}],"event":{"name":"Koli Calling '10: 10th Koli Calling International Conference on Computing Education Research","location":"Koli Finland","acronym":"Koli Calling '10","sponsor":["FU-BERLIN Free University Berlin","SIGCSE ACM Special Interest Group on Computer Science Education"]},"container-title":["Proceedings of the 10th Koli Calling International Conference on Computing Education Research"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1930464.1930480","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/1930464.1930480","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T12:08:19Z","timestamp":1750248499000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1930464.1930480"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2010,10,28]]},"references-count":80,"alternative-id":["10.1145\/1930464.1930480","10.1145\/1930464"],"URL":"https:\/\/doi.org\/10.1145\/1930464.1930480","relation":{},"subject":[],"published":{"date-parts":[[2010,10,28]]},"assertion":[{"value":"2010-10-28","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}