{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,1]],"date-time":"2025-11-01T17:47:05Z","timestamp":1762019225724,"version":"build-2065373602"},"publisher-location":"New York, NY, USA","reference-count":56,"publisher":"ACM","license":[{"start":{"date-parts":[[2019,11,21]],"date-time":"2019-11-21T00:00:00Z","timestamp":1574294400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["1836813, 1703304, 1735123, 1539179"],"award-info":[{"award-number":["1836813, 1703304, 1735123, 1539179"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2019,11,21]]},"DOI":"10.1145\/3364510.3364525","type":"proceedings-article","created":{"date-parts":[[2019,12,9]],"date-time":"2019-12-09T13:35:33Z","timestamp":1575898533000},"page":"1-10","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["Towards validity for a formative assessment for language-specific program tracing skills"],"prefix":"10.1145","author":[{"given":"Greg L.","family":"Nelson","sequence":"first","affiliation":[{"name":"University of Washington, Paul G. Allen School of Computer Science & Engineering, Seattle, Washington"}]},{"given":"Andrew","family":"Hu","sequence":"additional","affiliation":[{"name":"University of Washington, The Information School, DUB Group Seattle, Washington"}]},{"given":"Benjamin","family":"Xie","sequence":"additional","affiliation":[{"name":"University of Washington, The Information School, DUB Group Seattle, Washington"}]},{"given":"Amy J.","family":"Ko","sequence":"additional","affiliation":[{"name":"University of Washington, The Information School, DUB Group Seattle, Washington"}]}],"member":"320","published-online":{"date-parts":[[2019,11,21]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"SP","author":"Caceffo R.","year":"2018","unstructured":"Caceffo , R. , Gama , G. , Benatti , R. , Aparecida , T. , Caldas , T. , and Azevedo , R . A concept inventory for cs1 introductory programming courses in c. Tech. rep., University of Campinas , SP , Brasil , 2018 . Caceffo, R., Gama, G., Benatti, R., Aparecida, T., Caldas, T., and Azevedo, R. A concept inventory for cs1 introductory programming courses in c. Tech. rep., University of Campinas, SP, Brasil, 2018."},{"key":"e_1_3_2_1_2_1","first-page":"364","volume-title":"SIGCSE '16","author":"Caceffo R.","unstructured":"Caceffo , R. , Wolfman , S. , Booth , K. S. , and Azevedo , R . Developing a computer science concept inventory for introductory programming . SIGCSE '16 , ACM, pp. 364 -- 369 . Caceffo, R., Wolfman, S., Booth, K. S., and Azevedo, R. Developing a computer science concept inventory for introductory programming. SIGCSE '16, ACM, pp. 364--369."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_3_1","DOI":"10.1111\/medu.12678"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_4_1","DOI":"10.1145\/3160489.3160492"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_5_1","DOI":"10.1145\/3105726.3106190"},{"key":"e_1_3_2_1_6_1","first-page":"217","volume-title":"ICER (New York","author":"Danielsiek H.","year":"2017","unstructured":"Danielsiek , H. , Toma , L. , and Vahrenhold , J . An Instrument to Assess Self-Efficacy in Introductory Algorithms Courses . In ICER (New York , New York, USA , 2017 ), ACM Press , pp. 217 -- 225 . Danielsiek, H., Toma, L., and Vahrenhold, J. An Instrument to Assess Self-Efficacy in Introductory Algorithms Courses. In ICER (New York, New York, USA, 2017), ACM Press, pp. 217--225."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_8_1","DOI":"10.1145\/3287324.3287393"},{"key":"e_1_3_2_1_9_1","volume-title":"Proc. IFIP TC 34th World Conf. on Computers in Education WCEE 85","author":"Deimel L. E. J.","year":"1985","unstructured":"Deimel , L. E. J. , and Makoid , L . Developing program reading comprehension tests for the Computer Science classroom . Proc. IFIP TC 34th World Conf. on Computers in Education WCEE 85 ( 1985 ), 535--540. Deimel, L. E. J., and Makoid, L. Developing program reading comprehension tests for the Computer Science classroom. Proc. IFIP TC 34th World Conf. on Computers in Education WCEE 85 (1985), 535--540."},{"key":"e_1_3_2_1_10_1","first-page":"7","article-title":"A Critical Review of Research on Formative Assessment: The Limited Scientific Evidence of the Impact of Formative Assessment in Education. Practical Assessment","volume":"14","author":"Dunn K. E.","year":"2009","unstructured":"Dunn , K. E. , and Mulvenon , S. W . A Critical Review of Research on Formative Assessment: The Limited Scientific Evidence of the Impact of Formative Assessment in Education. Practical Assessment , Research & Evaluation 14 , 7 ( 2009 ). Dunn, K. E., and Mulvenon, S. W. A Critical Review of Research on Formative Assessment: The Limited Scientific Evidence of the Impact of Formative Assessment in Education. Practical Assessment, Research & Evaluation 14, 7 (2009).","journal-title":"Research & Evaluation"},{"key":"e_1_3_2_1_11_1","first-page":"121","volume-title":"Exploring the Value of Student Self-Evaluation in Introductory Programming. In ICER '19","author":"Duran R.","year":"2019","unstructured":"Duran , R. , Rybicki , J.-M. , Sorva , J. , and Hellas , A . Exploring the Value of Student Self-Evaluation in Introductory Programming. In ICER '19 ( 2019 ), ACM Press , pp. 121 -- 130 . Duran, R., Rybicki, J.-M., Sorva, J., and Hellas, A. Exploring the Value of Student Self-Evaluation in Introductory Programming. In ICER '19 (2019), ACM Press, pp. 121--130."},{"key":"e_1_3_2_1_13_1","first-page":"60","volume-title":"Evaluating the Efficiency and Effectiveness of Adaptive Parsons Problems. In ICER '18","author":"Ericson B. J.","year":"2018","unstructured":"Ericson , B. J. , Foley , J. D. , and Rick , J . Evaluating the Efficiency and Effectiveness of Adaptive Parsons Problems. In ICER '18 ( 2018 ), ACM Press , pp. 60 -- 68 . Ericson, B. J., Foley, J. D., and Rick, J. Evaluating the Efficiency and Effectiveness of Adaptive Parsons Problems. In ICER '18 (2018), ACM Press, pp. 60--68."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_14_1","DOI":"10.1145\/2858796.2858801"},{"key":"e_1_3_2_1_15_1","volume-title":"14th Australasian Computing Education Conference","author":"Gluga R.","year":"2012","unstructured":"Gluga , R. , Kay , J. , Lister , R. , Kleitman , S. , and Lever , T . Coming to terms with Bloom: an online tutorial for teachers of programming fundamentals . 14th Australasian Computing Education Conference ( 2012 ), 147--156. Gluga, R., Kay, J., Lister, R., Kleitman, S., and Lever, T. Coming to terms with Bloom: an online tutorial for teachers of programming fundamentals. 14th Australasian Computing Education Conference (2012), 147--156."},{"key":"e_1_3_2_1_16_1","volume-title":"Influences on student Learning. Tech. rep","author":"Hattie J.","year":"1999","unstructured":"Hattie , J. Influences on student Learning. Tech. rep ., 1999 . Hattie, J. Influences on student Learning. Tech. rep., 1999."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_17_1","DOI":"10.3102\/003465430298487"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_18_1","DOI":"10.1145\/369295.369324"},{"key":"e_1_3_2_1_19_1","first-page":"4","article-title":"Current Concerns","volume":"38","author":"Kane M. J","year":"2016","unstructured":"Kane , M. J . Current Concerns in Validity Theory. Language Learning 38 , 4 ( 2016 ), 319--342. Kane, M. J. Current Concerns in Validity Theory. Language Learning 38, 4 (2016), 319--342.","journal-title":"Validity Theory. Language Learning"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_20_1","DOI":"10.1111\/jedm.12000"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_21_1","DOI":"10.1111\/jedm.12000"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_22_1","DOI":"10.1145\/3279720.3279728"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_23_1","DOI":"10.1111\/j.1745-3992.2011.00220.x"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_24_1","DOI":"10.1080\/00011037.1980.11008280"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_25_1","DOI":"10.1145\/2978249.2978251"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_26_1","DOI":"10.1145\/1044550.1041673"},{"key":"e_1_3_2_1_27_1","first-page":"26","volume-title":"The Compound Nature of Novice Programming Assessments. In Proceedings of the Nineteenth Australasian Computing Education Conference on - ACE '17","author":"Luxton-Reilly A.","year":"2017","unstructured":"Luxton-Reilly , A. , and Petersen , A . The Compound Nature of Novice Programming Assessments. In Proceedings of the Nineteenth Australasian Computing Education Conference on - ACE '17 ( 2017 ), ACM Press , pp. 26 -- 35 . Luxton-Reilly, A., and Petersen, A. The Compound Nature of Novice Programming Assessments. In Proceedings of the Nineteenth Australasian Computing Education Conference on - ACE '17 (2017), ACM Press, pp. 26--35."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_28_1","DOI":"10.1145\/3174781.3174784"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_29_1","DOI":"10.1145\/572139.572181"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_30_1","DOI":"10.1145\/1189215.1189185"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_31_1","DOI":"10.1037\/0003-066X.50.9.741"},{"key":"e_1_3_2_1_32_1","first-page":"109","volume":"30","author":"Moss P. A.","year":"2006","unstructured":"Moss , P. A. , Girard , B. J. , and Haniford , L. C . Validity in Educational Assessment. Review of Research in Education 30 ( 2006 ), 109 -- 162 . Moss, P. A., Girard, B. J., and Haniford, L. C. Validity in Educational Assessment. Review of Research in Education 30 (2006), 109--162.","journal-title":"Validity in Educational Assessment. Review of Research in Education"},{"key":"e_1_3_2_1_33_1","volume-title":"Comprehension First: Evaluating a Novel Pedagogy and Tutoring System for Program Tracing in CS1. ICER","author":"Nelson G. L.","year":"2017","unstructured":"Nelson , G. L. , Xie , B. , and Ko , A. J . Comprehension First: Evaluating a Novel Pedagogy and Tutoring System for Program Tracing in CS1. ICER ( 2017 ). Nelson, G. L., Xie, B., and Ko, A. J. Comprehension First: Evaluating a Novel Pedagogy and Tutoring System for Program Tracing in CS1. ICER (2017)."},{"key":"e_1_3_2_1_34_1","volume-title":"Replication, validation, and use of a language independent CS1 knowledge assessment. ICER","author":"Parker M. C.","year":"2016","unstructured":"Parker , M. C. , and Guzdial , M . Replication, validation, and use of a language independent CS1 knowledge assessment. ICER ( 2016 ), 93--101. Parker, M. C., and Guzdial, M. Replication, validation, and use of a language independent CS1 knowledge assessment. ICER (2016), 93--101."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_35_1","DOI":"10.1145\/2960310.2960316"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_36_1","DOI":"10.2190\/689T-1R2A-X4W4-29J2"},{"key":"e_1_3_2_1_37_1","volume-title":"Fragile Knowledge and Neglected Strategies in Novice Programmers. IR85-22. Tech. rep","author":"Perkins D.","year":"1985","unstructured":"Perkins , D. , and Martin , F . Fragile Knowledge and Neglected Strategies in Novice Programmers. IR85-22. Tech. rep ., 1985 . Perkins, D., and Martin, F. Fragile Knowledge and Neglected Strategies in Novice Programmers. IR85-22. Tech. rep., 1985."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_38_1","DOI":"10.1145\/1953163.1953340"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_39_1","DOI":"10.1145\/3159450.3159587"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_40_1","DOI":"10.1111\/j.1745-3992.2010.00189.x"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_41_1","DOI":"10.1145\/3291279.3339404"},{"key":"e_1_3_2_1_42_1","volume-title":"J. C. Online Assessment of SQL Query Formulation Skills. Ace '03 20","author":"Prior","year":"2003","unstructured":"Prior , J. C. Online Assessment of SQL Query Formulation Skills. Ace '03 20 ( 2003 ), 247--256. Prior, J. C. Online Assessment of SQL Query Formulation Skills. Ace '03 20 (2003), 247--256."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_43_1","DOI":"10.1109\/LaTiCE.2017.13"},{"key":"e_1_3_2_1_44_1","first-page":"33","volume-title":"The Canterbury QuestionBank. In ITiCSE -WGR '13","author":"Sanders K.","year":"2013","unstructured":"Sanders , K. , Spacco , J. , Ahmadzadeh , M. , Clear , T. , Edwards , S. H. , Goldweber , M. , Johnson , C. , Lister , R. , McCartney , R. , and Patitsas , E . The Canterbury QuestionBank. In ITiCSE -WGR '13 ( 2013 ), ACM Press , pp. 33 -- 52 . Sanders, K., Spacco, J., Ahmadzadeh, M., Clear, T., Edwards, S. H., Goldweber, M., Johnson, C., Lister, R., McCartney, R., and Patitsas, E. The Canterbury QuestionBank. In ITiCSE -WGR '13 (2013), ACM Press, pp. 33--52."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_45_1","DOI":"10.1145\/1971681.1971687"},{"key":"e_1_3_2_1_46_1","volume-title":"Benchmarking Introductory Programming Exams: Some Preliminary Results. ICER '16","author":"Sheard J.","year":"2016","unstructured":"Sheard , J. , Souza , D. D. , Klemperer , P. , Porter , L. , and Zingaro , D . Benchmarking Introductory Programming Exams: Some Preliminary Results. ICER '16 ( 2016 ), 103--111. Sheard, J., Souza, D. D., Klemperer, P., Porter, L., and Zingaro, D. Benchmarking Introductory Programming Exams: Some Preliminary Results. ICER '16 (2016), 103--111."},{"key":"e_1_3_2_1_47_1","first-page":"154","volume-title":"Benchmarking Introductory Programming Exams. In ITiCSE '16","volume":"159","author":"Simon","year":"2016","unstructured":"Simon , Sheard, J., D'Souza , D. , Klemperer , P. , Porter , L. , Sorva , J. , Stegeman , M. , and Zingaro , D . Benchmarking Introductory Programming Exams. In ITiCSE '16 ( 2016 ), vol. 159 , ACM Press , pp. 154 -- 159 . Simon, Sheard, J., D'Souza, D., Klemperer, P., Porter, L., Sorva, J., Stegeman, M., and Zingaro, D. Benchmarking Introductory Programming Exams. In ITiCSE '16 (2016), vol. 159, ACM Press, pp. 154--159."},{"key":"e_1_3_2_1_48_1","first-page":"209","volume-title":"Principled Assessment of Student Learning in High School Computer Science. In ICER '17","author":"Snow E.","year":"2017","unstructured":"Snow , E. , Rutstein , D. , Bienkowski , M. , and Xu , Y . Principled Assessment of Student Learning in High School Computer Science. In ICER '17 ( 2017 ), ACM Press , pp. 209 -- 216 . Snow, E., Rutstein, D., Bienkowski, M., and Xu, Y. Principled Assessment of Student Learning in High School Computer Science. In ICER '17 (2017), ACM Press, pp. 209--216."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_49_1","DOI":"10.1145\/169070.169109"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_50_1","DOI":"10.1080\/08993408.2014.970779"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_51_1","DOI":"10.1145\/1734263.1734297"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_52_1","DOI":"10.1145\/1953163.1953200"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_53_1","DOI":"10.1145\/2543882.2543884"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_54_1","DOI":"10.1145\/3287324.3287370"},{"unstructured":"Xie B. Loksa D. Nelson G. L. Davidson M. J. Dong D. Kwik H. Tan A. H. Hwa L. Li M. and Ko A. J. A theory of instruction for introductory programming skills. Computer Science Education (2019) 1--49.  Xie B. Loksa D. Nelson G. L. Davidson M. J. Dong D. Kwik H. Tan A. H. Hwa L. Li M. and Ko A. J. A theory of instruction for introductory programming skills. Computer Science Education (2019) 1--49.","key":"e_1_3_2_1_55_1"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_56_1","DOI":"10.1145\/3159450.3159527"},{"unstructured":"Zingaro D. and Petersen A. Stepping Up to Integrative Questions on CS1 Exams. 253--258.  Zingaro D. and Petersen A. Stepping Up to Integrative Questions on CS1 Exams. 253--258.","key":"e_1_3_2_1_57_1"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_58_1","DOI":"10.1145\/2462476.2462479"}],"event":{"sponsor":["University of Helsinki","Univ. Eastern Finland University of Eastern Finland","The University of Adelaide, Australia","SIGCSE ACM Special Interest Group on Computer Science Education"],"acronym":"Koli Calling '19","name":"Koli Calling '19: 19th Koli Calling International Conference on Computing Education Research","location":"Koli Finland"},"container-title":["Proceedings of the 19th Koli Calling International Conference on Computing Education Research"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3364510.3364525","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3364510.3364525","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3364510.3364525","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T23:23:23Z","timestamp":1750202603000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3364510.3364525"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,11,21]]},"references-count":56,"alternative-id":["10.1145\/3364510.3364525","10.1145\/3364510"],"URL":"https:\/\/doi.org\/10.1145\/3364510.3364525","relation":{},"subject":[],"published":{"date-parts":[[2019,11,21]]},"assertion":[{"value":"2019-11-21","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}