{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,8]],"date-time":"2026-02-08T01:00:45Z","timestamp":1770512445154,"version":"3.49.0"},"reference-count":39,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2021,4,23]],"date-time":"2021-04-23T00:00:00Z","timestamp":1619136000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,4,23]],"date-time":"2021-04-23T00:00:00Z","timestamp":1619136000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Educ Inf Technol"],"published-print":{"date-parts":[[2021,9]]},"DOI":"10.1007\/s10639-021-10553-9","type":"journal-article","created":{"date-parts":[[2021,4,25]],"date-time":"2021-04-25T03:23:09Z","timestamp":1619320989000},"page":"5477-5498","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["Levels of problem-solving competency identified through Bebras Computing Challenge"],"prefix":"10.1007","volume":"26","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8646-0144","authenticated-orcid":false,"given":"Kyungbin","family":"Kwon","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jongpil","family":"Cheon","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hyunchang","family":"Moon","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2021,4,23]]},"reference":[{"key":"10553_CR1","doi-asserted-by":"publisher","first-page":"30","DOI":"10.1016\/j.ijcci.2018.10.004","volume":"19","author":"Y Allsop","year":"2019","unstructured":"Allsop, Y. (2019). Assessing computational thinking process using a multiple evaluation approach. International Journal of Child-Computer Interaction, 19, 30\u201355. https:\/\/doi.org\/10.1016\/j.ijcci.2018.10.004","journal-title":"International Journal of Child-Computer Interaction"},{"key":"10553_CR2","volume-title":"Investigating student generated computational models of science","author":"S Basu","year":"2014","unstructured":"Basu, S., Dukeman, A., Kinnebrew, J. S., Biswas, G., & Sengupta, P. (2014). Investigating student generated computational models of science. Paper presented at the International Society of the Learning Sciences."},{"key":"10553_CR3","unstructured":"Bhagat, K. K., & Spector, J. M. (2017). Formative assessment in complex problem-solving domains: The emerging role of assessment technologies. Journal of Educational Technology & Society, 20(4), 312\u2013317. Retrieved April 22, 2021, http:\/\/www.jstor.org\/stable\/26229226"},{"key":"10553_CR4","volume-title":"The IDEAL problem solver","author":"JD Bransford","year":"1993","unstructured":"Bransford, J. D., & Stein, B. S. (1993). The IDEAL problem solver. (2nd ed.). Freeman.","edition":"2"},{"key":"10553_CR5","unstructured":"Brush, T., Ottenbreit-Leftwich, A., Kwon, K., & Karlin, M. (2020). Implementing Socially Relevant Problem-Based Computer Science Curriculum at the Elementary Level: Students\u2019 Computer Science Knowledge and Teachers\u2019 Implementation Needs. Journal of Computers in Mathematics and Science Teaching, 39(2), 109\u2013123. Retrieved April 22, 2021, from https:\/\/www.learntechlib.org\/primary\/p\/210969\/"},{"key":"10553_CR6","doi-asserted-by":"publisher","first-page":"167","DOI":"10.1017\/CBO9780511816796.010","volume-title":"The Cambridge handbook of expertise and expert performance","author":"MTH Chi","year":"2006","unstructured":"Chi, M. T. H. (2006). Laboratory methods for assessing experts\u2019 and novices\u2019 knowledge. In K. Ericsson, N. Charness, P. J. Feltovich, & R. R. Hoffman (Eds.), The Cambridge handbook of expertise and expert performance. (pp. 167\u2013184). Cambridge University Press."},{"issue":"2","key":"10553_CR7","doi-asserted-by":"publisher","first-page":"121","DOI":"10.1207\/s15516709cog0502_2","volume":"5","author":"MTH Chi","year":"1981","unstructured":"Chi, M. T. H., Feltovich, P. J., & Glaser, R. (1981). Categorization and representation of physics problems by experts and novices. Cognitive Science, 5(2), 121\u2013152. https:\/\/doi.org\/10.1207\/s15516709cog0502_2","journal-title":"Cognitive Science"},{"key":"10553_CR8","doi-asserted-by":"publisher","unstructured":"Chiazzese, G., Arrigo, M., Chifari, A., Lonati, V., & Tosto, C. (2019). Educational robotics in primary school: measuring the development of computational thinking skills with the Bebras tasks. Informatics, 6(4), 43. Retrieved from\u00a0https:\/\/doi.org\/10.3390\/informatics6040043.","DOI":"10.3390\/informatics6040043"},{"key":"10553_CR9","first-page":"19","volume-title":"International conference on informatics in secondary schools-evolution and perspectives","author":"V Dagien\u0117","year":"2008","unstructured":"Dagien\u0117, V., & Futschek, G. (2008). Bebras international contest on informatics and computer literacy: Criteria for good tasks. In R. T. Mittermeir & M. M. Sys\u0142o (Eds.), International conference on informatics in secondary schools-evolution and perspectives. (pp. 19\u201330). Springer."},{"issue":"1","key":"10553_CR10","doi-asserted-by":"publisher","first-page":"25","DOI":"10.15388\/infedu.2016.02","volume":"15","author":"V Dagien\u0117","year":"2016","unstructured":"Dagien\u0117, V., & Stupuriene, G. (2016). Bebras\u2013a sustainable community building model for the concept based learning of informatics and computational thinking. Informatics in education, 15(1), 25\u201344. https:\/\/doi.org\/10.15388\/infedu.2016.02","journal-title":"Informatics in education"},{"key":"10553_CR11","unstructured":"Dolgopolovas, V., Jevsikova, T., Savulionien\u0117, L., Dagien\u0117, V. (2015) On evaluation of computational thinking of software engineering novice students. Proceedings of the IFIP TC3 working conference. A new culture of learning: computing and next generations, 90\u201399."},{"key":"10553_CR12","doi-asserted-by":"publisher","DOI":"10.1145\/2818314.2818328","volume-title":"A pilot computer science and programming course for primary school students","author":"C Duncan","year":"2015","unstructured":"Duncan, C., & Bell, T. (2015). A pilot computer science and programming course for primary school students. Paper presented at the Workshop in Primary and Secondary Computing Education."},{"issue":"2","key":"10553_CR13","doi-asserted-by":"publisher","first-page":"157","DOI":"10.1348\/000709904X23411","volume":"75","author":"LM Fawcett","year":"2005","unstructured":"Fawcett, L. M., & Garton, A. F. (2005). The effect of peer collaboration on children\u2019s problem-solving ability. British Journal of Educational Psychology, 75(2), 157\u2013169. https:\/\/doi.org\/10.1348\/000709904X23411","journal-title":"British Journal of Educational Psychology"},{"key":"10553_CR14","doi-asserted-by":"publisher","unstructured":"Fees, R. E., Rosa, J. A. d., Durkin, S. S., Murray, M. M., & Moran, A. L. (2018). Unplugged Cybersecurity: An approach for bringing computer science into the classroom. International Journal of Computer Science Education in Schools, 2(1). https:\/\/doi.org\/10.21585\/ijcses.v2i1.21","DOI":"10.21585\/ijcses.v2i1.21"},{"issue":"4","key":"10553_CR15","doi-asserted-by":"publisher","first-page":"365","DOI":"10.2307\/356600","volume":"32","author":"L Flower","year":"1981","unstructured":"Flower, L., & Hayes, J. R. (1981). A cognitive process theory of writing. College composition and communication, 32(4), 365\u2013387. https:\/\/doi.org\/10.2307\/356600","journal-title":"College composition and communication"},{"issue":"4","key":"10553_CR16","doi-asserted-by":"publisher","first-page":"552","DOI":"10.1080\/03043797.2017.1299692","volume":"43","author":"B Frank","year":"2018","unstructured":"Frank, B., Simper, N., & Kaupp, J. (2018). Formative feedback and scaffolding for developing complex problem solving and modelling outcomes. European Journal of Engineering Education, 43(4), 552\u2013568. https:\/\/doi.org\/10.1080\/03043797.2017.1299692","journal-title":"European Journal of Engineering Education"},{"issue":"1\u20132","key":"10553_CR17","doi-asserted-by":"publisher","first-page":"99","DOI":"10.1080\/00461520.1986.9653026","volume":"21","author":"ML Gick","year":"1986","unstructured":"Gick, M. L. (1986). Problem-Solving Strategies. Educational Psychologist, 21(1\u20132), 99\u2013120. https:\/\/doi.org\/10.1080\/00461520.1986.9653026","journal-title":"Educational Psychologist"},{"key":"10553_CR18","doi-asserted-by":"publisher","first-page":"74","DOI":"10.1016\/j.edurev.2014.10.002","volume":"13","author":"S Greiff","year":"2014","unstructured":"Greiff, S., W\u00fcstenberg, S., Csap\u00f3, B., Demetriou, A., Hautam\u00e4ki, J., Graesser, A. C., & Martin, R. (2014). Domain-general problem solving skills and education in the 21st century. Educational Research Review, 13, 74\u201383. https:\/\/doi.org\/10.1016\/j.edurev.2014.10.002","journal-title":"Educational Research Review"},{"issue":"1","key":"10553_CR19","doi-asserted-by":"publisher","first-page":"127","DOI":"10.1207\/s15516709cog2801_7","volume":"28","author":"CE Hmelo-Silver","year":"2004","unstructured":"Hmelo-Silver, C. E., & Pfeffer, M. G. (2004). Comparing expert and novice understanding of a complex system from the perspective of structures, behaviors, and functions. Cognitive Science, 28(1), 127\u2013138. https:\/\/doi.org\/10.1207\/s15516709cog2801_7","journal-title":"Cognitive Science"},{"key":"10553_CR20","doi-asserted-by":"publisher","first-page":"1277","DOI":"10.1177\/1049732305276687","volume":"15","author":"H-F Hsieh","year":"2005","unstructured":"Hsieh, H.-F., & Shannon, S. E. (2005). Three approaches to qualitative content analysis. Qualitative Health Research, 15, 1277\u20131288. https:\/\/doi.org\/10.1177\/1049732305276687","journal-title":"Qualitative Health Research"},{"key":"10553_CR21","doi-asserted-by":"crossref","unstructured":"Kalelio\u011flu, F., G\u00fclbahar, Y., & Madran, O. (2015). A Snapshot of the First Implementation of Bebras International Informatics Contest in Turkey. Paper presented at the International Conference on Informatics in Schools: Situation, Evolution, and Perspectives, Cham.","DOI":"10.1007\/978-3-319-25396-1_12"},{"issue":"2","key":"10553_CR22","doi-asserted-by":"publisher","first-page":"289","DOI":"10.1080\/00461520.2016.1155457","volume":"51","author":"M Kapur","year":"2016","unstructured":"Kapur, M. (2016). Examining Productive Failure, Productive Success, Unproductive Failure, and Unproductive Success in Learning. Educational Psychologist, 51(2), 289\u2013299. https:\/\/doi.org\/10.1080\/00461520.2016.1155457","journal-title":"Educational Psychologist"},{"issue":"1","key":"10553_CR23","doi-asserted-by":"publisher","first-page":"45","DOI":"10.1080\/10508406.2011.591717","volume":"21","author":"M Kapur","year":"2012","unstructured":"Kapur, M., & Bielaczyc, K. (2012). Designing for Productive Failure. Journal of the Learning Sciences, 21(1), 45\u201383. https:\/\/doi.org\/10.1080\/10508406.2011.591717","journal-title":"Journal of the Learning Sciences"},{"issue":"4","key":"10553_CR24","doi-asserted-by":"publisher","first-page":"222","DOI":"10.1159\/000272885","volume":"26","author":"KS Kitchner","year":"1983","unstructured":"Kitchner, K. S. (1983). Cognition, Metacognition, and Epistemic Cognition. Human Development, 26(4), 222\u2013232. https:\/\/doi.org\/10.1159\/000272885","journal-title":"Human Development"},{"key":"10553_CR25","doi-asserted-by":"publisher","first-page":"558","DOI":"10.1016\/j.chb.2017.01.005","volume":"72","author":"\u00d6 Korkmaz","year":"2017","unstructured":"Korkmaz, \u00d6., \u00c7akir, R., & \u00d6zden, M. Y. (2017). A validity and reliability study of the computational thinking scales (CTS). Computers in Human Behavior, 72, 558\u2013569. https:\/\/doi.org\/10.1016\/j.chb.2017.01.005","journal-title":"Computers in Human Behavior"},{"issue":"2","key":"10553_CR26","doi-asserted-by":"publisher","first-page":"103","DOI":"10.1177\/0022219412436976","volume":"47","author":"JL Krawec","year":"2014","unstructured":"Krawec, J. L. (2014). Problem Representation and Mathematical Problem Solving of Students of Varying Math Ability. Journal of Learning Disabilities, 47(2), 103\u2013115. https:\/\/doi.org\/10.1177\/0022219412436976","journal-title":"Journal of Learning Disabilities"},{"issue":"6","key":"10553_CR27","first-page":"45","volume":"30","author":"EA Lamagna","year":"2015","unstructured":"Lamagna, E. A. (2015). Algorithmic thinking unplugged. Journal of Computing Sciences in Colleges, 30(6), 45\u201352","journal-title":"Journal of Computing Sciences in Colleges"},{"issue":"4","key":"10553_CR28","doi-asserted-by":"publisher","first-page":"317","DOI":"10.1016\/S0364-0213(80)80008-5","volume":"4","author":"JH Larkin","year":"1980","unstructured":"Larkin, J. H., McDermott, J., Simon, D. P., & Simon, H. A. (1980). Models of competence in solving physics problems. Cognitive Science, 4(4), 317\u2013345. https:\/\/doi.org\/10.1016\/S0364-0213(80)80008-5","journal-title":"Cognitive Science"},{"key":"10553_CR29","doi-asserted-by":"publisher","DOI":"10.1515\/9781400828678","volume-title":"How to solve it","author":"G Polya","year":"1945","unstructured":"Polya, G. (1945). How to solve it. Princeton University Press."},{"issue":"5","key":"10553_CR30","doi-asserted-by":"publisher","first-page":"2234","DOI":"10.1111\/bjet.12843","volume":"50","author":"CM Reilly","year":"2019","unstructured":"Reilly, C. M., Kang, S. Y., Grotzer, T. A., Joyal, J. A., & Oriol, N. E. (2019). Pedagogical moves and student thinking in technology-mediated medical problem-based learning: Supporting novice-expert shift. British Journal of Educational Technology, 50(5), 2234\u20132250. https:\/\/doi.org\/10.1111\/bjet.12843","journal-title":"British Journal of Educational Technology"},{"key":"10553_CR31","doi-asserted-by":"publisher","first-page":"91","DOI":"10.1017\/CBO9780511844744.007","volume-title":"Cognitive load theory","author":"A Renkl","year":"2010","unstructured":"Renkl, A., & Atkinson, R. K. (2010). Learning from worked-out examples and problem solving. Cognitive load theory. (pp. 91\u2013108). Cambridge University Press."},{"key":"10553_CR32","doi-asserted-by":"publisher","first-page":"68","DOI":"10.18293\/VLSS2017-010","volume":"3","author":"A Repenning","year":"2017","unstructured":"Repenning, A. (2017). Moving Beyond Syntax: Lessons from 20 Years of Blocks Programing in AgentSheets. Journal of Visual Languages and Sentient Systems, 3, 68\u201391. https:\/\/doi.org\/10.18293\/VLSS2017-010","journal-title":"Journal of Visual Languages and Sentient Systems"},{"key":"10553_CR33","doi-asserted-by":"publisher","first-page":"678","DOI":"10.1016\/j.chb.2016.08.047","volume":"72","author":"M Rom\u00e1n-Gonz\u00e1lez","year":"2017","unstructured":"Rom\u00e1n-Gonz\u00e1lez, M., P\u00e9rez-Gonz\u00e1lez, J.-C., & Jim\u00e9nez-Fern\u00e1ndez, C. (2017). Which cognitive abilities underlie computational thinking? Criterion validity of the Computational Thinking Test. Computers in Human Behavior, 72, 678\u2013691. https:\/\/doi.org\/10.1016\/j.chb.2016.08.047","journal-title":"Computers in Human Behavior"},{"issue":"4","key":"10553_CR34","doi-asserted-by":"publisher","first-page":"207","DOI":"10.1111\/j.1467-9280.1992.tb00029.x","volume":"3","author":"RA Schmidt","year":"1992","unstructured":"Schmidt, R. A., & Bjork, R. A. (1992). New conceptualizations of practice: Common principles in three paradigms suggest new concepts for training. Psychological Science, 3(4), 207\u2013218. https:\/\/doi.org\/10.1111\/j.1467-9280.1992.tb00029.x","journal-title":"Psychological Science"},{"key":"10553_CR35","volume-title":"Mathematical problem-solving","author":"AH Schoenfeld","year":"1985","unstructured":"Schoenfeld, A. H. (1985). Mathematical problem-solving. Academic Press."},{"issue":"3","key":"10553_CR36","doi-asserted-by":"publisher","first-page":"107","DOI":"10.1016\/j.cogpsych.2011.06.001","volume":"63","author":"VA Thompson","year":"2011","unstructured":"Thompson, V. A., Prowse Turner, J. A., & Pennycook, G. (2011). Intuition, reason, and metacognition. Cognitive Psychology, 63(3), 107\u2013140. https:\/\/doi.org\/10.1016\/j.cogpsych.2011.06.001","journal-title":"Cognitive Psychology"},{"key":"10553_CR37","volume-title":"Mind in society","author":"LS Vygotsky","year":"1978","unstructured":"Vygotsky, L. S. (1978). Mind in society. Harvard University Press."},{"issue":"1","key":"10553_CR38","doi-asserted-by":"publisher","first-page":"81","DOI":"10.1016\/j.cogsys.2008.08.003","volume":"11","author":"Y Wang","year":"2010","unstructured":"Wang, Y., & Chiew, V. (2010). On the cognitive process of human problem solving. Cognitive Systems Research, 11(1), 81\u201392. https:\/\/doi.org\/10.1016\/j.cogsys.2008.08.003","journal-title":"Cognitive Systems Research"},{"issue":"3","key":"10553_CR39","doi-asserted-by":"publisher","first-page":"33","DOI":"10.1145\/1118178.1118215","volume":"49","author":"JM Wing","year":"2006","unstructured":"Wing, J. M. (2006). Computational thinking. Communications of the ACM, 49(3), 33\u201335. https:\/\/doi.org\/10.1145\/1118178.1118215","journal-title":"Communications of the ACM"}],"container-title":["Education and Information Technologies"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10639-021-10553-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10639-021-10553-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10639-021-10553-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,8,27]],"date-time":"2021-08-27T15:42:39Z","timestamp":1630078959000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10639-021-10553-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,4,23]]},"references-count":39,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2021,9]]}},"alternative-id":["10553"],"URL":"https:\/\/doi.org\/10.1007\/s10639-021-10553-9","relation":{},"ISSN":["1360-2357","1573-7608"],"issn-type":[{"value":"1360-2357","type":"print"},{"value":"1573-7608","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,4,23]]},"assertion":[{"value":"10 February 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"18 April 2021","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 April 2021","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The study was approved by the University Institutional Review Board (IRB2018-1056) and did not involve any monetary compensation.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics approval"}},{"value":"No potential competing interest was reported by the authors.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}]}}