{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,28]],"date-time":"2026-07-28T12:20:46Z","timestamp":1785241246611,"version":"3.55.0"},"reference-count":55,"publisher":"Elsevier BV","issue":"2","license":[{"start":{"date-parts":[[2025,6,1]],"date-time":"2025-06-01T00:00:00Z","timestamp":1748736000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2025,6,1]],"date-time":"2025-06-01T00:00:00Z","timestamp":1748736000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,4,22]],"date-time":"2026-04-22T00:00:00Z","timestamp":1776816000000},"content-version":"vor","delay-in-days":325,"URL":"http:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["International Journal of Artificial Intelligence in Education"],"published-print":{"date-parts":[[2025,6]]},"DOI":"10.1007\/s40593-024-00406-0","type":"journal-article","created":{"date-parts":[[2024,5,15]],"date-time":"2024-05-15T16:19:30Z","timestamp":1715789970000},"page":"774-790","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":44,"title":["Evaluation of LLM Tools for Feedback Generation in a Course on Concurrent Programming"],"prefix":"10.1016","volume":"35","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1047-5398","authenticated-orcid":false,"given":"Iria","family":"Est\u00e9vez-Ayres","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6124-6213","authenticated-orcid":false,"given":"Patricia","family":"Callejo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8254-8795","authenticated-orcid":false,"given":"Miguel \u00c1ngel","family":"Hombrados-Herrera","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3082-0814","authenticated-orcid":false,"given":"Carlos","family":"Alario-Hoyos","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4093-3705","authenticated-orcid":false,"given":"Carlos","family":"Delgado Kloos","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1007\/s40593-024-00406-0_bib1","doi-asserted-by":"crossref","first-page":"723447","DOI":"10.3389\/frai.2021.723447","article-title":"Explainable ai for data-driven feedback and intelligent action recommendations to support students self-regulation","volume":"4","author":"Afzaal","year":"2021","journal-title":"Frontiers in Artificial Intelligence."},{"issue":"2","key":"10.1007\/s40593-024-00406-0_bib2","doi-asserted-by":"crossref","first-page":"83","DOI":"10.1080\/08993400500150747","article-title":"A survey of automated assessment approaches for programming assignments","volume":"15","author":"Ala-Mutka","year":"2005","journal-title":"Computer Science Education"},{"key":"10.1007\/s40593-024-00406-0_bib3","series-title":"Formal Techniques for Distributed Objects, Components, and Systems","first-page":"3","article-title":"An experimental evaluation of tools for grading concurrent programming exercises","author":"Barros","year":"2023"},{"key":"10.1007\/s40593-024-00406-0_bib4","doi-asserted-by":"crossref","DOI":"10.1145\/3276514","article-title":"Racerd: compositional static race detection","volume":"2","author":"Blackshear","year":"2018","journal-title":"Proceedings of the ACM on Programming Languages"},{"issue":"3","key":"10.1007\/s40593-024-00406-0_bib5","doi-asserted-by":"crossref","first-page":"245","DOI":"10.3102\/00346543065003245","article-title":"Feedback and self-regulated learning: A theoretical synthesis","volume":"65","author":"Butler","year":"1995","journal-title":"Review of Educational Research"},{"key":"10.1007\/s40593-024-00406-0_bib6","series-title":"NASA Formal Methods","first-page":"3","article-title":"Moving fast with software verification","author":"Calcagno","year":"2015"},{"key":"10.1007\/s40593-024-00406-0_bib7","series-title":"Attention and Self-regulation: A Control-theory Approach to Human Behavior.","author":"Carver","year":"2012"},{"key":"10.1007\/s40593-024-00406-0_bib8","first-page":"100027","article-title":"Automatic feedback in online learning environments: A systematic literature review","volume":"2","author":"Cavalcanti","year":"2021","journal-title":"Computers and Education: Artificial Intelligence."},{"key":"10.1007\/s40593-024-00406-0_bib9","doi-asserted-by":"crossref","first-page":"17754","DOI":"10.1609\/aaai.v38i16.29728","article-title":"Benchmarking large language models in retrieval-augmented generation","volume":"38","author":"Chen","year":"2024","journal-title":"Proceedings of the AAAI Conference on Artificial Intelligence"},{"issue":"1","key":"10.1007\/s40593-024-00406-0_bib10","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/s41239-020-00233-y","article-title":"An analysis of internal and external feedback in self-regulated learning activities mediated by self-regulated learning tools and open learner models","volume":"17","author":"Chou","year":"2020","journal-title":"International Journal of Educational Technology in Higher Education"},{"issue":"6","key":"10.1007\/s40593-024-00406-0_bib11","doi-asserted-by":"crossref","first-page":"763","DOI":"10.1017\/S1351324921000322","article-title":"Emerging trends: A gentle introduction to fine-tuning","volume":"27","author":"Church","year":"2021","journal-title":"Natural Language Engineering"},{"key":"10.1007\/s40593-024-00406-0_bib12","doi-asserted-by":"crossref","unstructured":"Dai, W., Lin, J., Jin, F., Li, T., Tsai, Y.-S., Gasevic, D., Chen, G. (2023). Can large language models provide feedback to students? a case study on chatgpt.","DOI":"10.35542\/osf.io\/hcgzj"},{"key":"10.1007\/s40593-024-00406-0_bib13","doi-asserted-by":"crossref","first-page":"104094","DOI":"10.1016\/j.compedu.2020.104094","article-title":"A review of automated feedback systems for learners: Classification framework, challenges and opportunities","volume":"162","author":"Deeva","year":"2021","journal-title":"Computers & Education."},{"issue":"3","key":"10.1007\/s40593-024-00406-0_bib14","doi-asserted-by":"crossref","first-page":"4","DOI":"10.1145\/1163405.1163409","article-title":"Automatic test-based assessment of programming: A review","volume":"5","author":"Douce","year":"2005","journal-title":"Journal on Educational Resources in Computing (JERIC)."},{"issue":"1","key":"10.1007\/s40593-024-00406-0_bib15","doi-asserted-by":"crossref","first-page":"38","DOI":"10.1017\/S0376892997000088","article-title":"A review of methods for the assessment of prediction errors in conservation presence\/absence models","volume":"24","author":"Fielding","year":"1997","journal-title":"Environmental Conservation"},{"key":"10.1007\/s40593-024-00406-0_bib16","doi-asserted-by":"crossref","unstructured":"Gao, A. (2023). Prompt engineering for large language models. Available at SSRN 4504303.","DOI":"10.2139\/ssrn.4504303"},{"issue":"1","key":"10.1007\/s40593-024-00406-0_bib17","doi-asserted-by":"crossref","first-page":"81","DOI":"10.3102\/003465430298487","article-title":"The power of feedback","volume":"77","author":"Hattie","year":"2007","journal-title":"Review of Educational Research"},{"issue":"3","key":"10.1007\/s40593-024-00406-0_bib18","doi-asserted-by":"crossref","first-page":"179","DOI":"10.1145\/356603.356607","article-title":"Some deadlock properties of computer systems","volume":"4","author":"Holt","year":"1972","journal-title":"ACM Computing Surveys (CSUR)."},{"key":"10.1007\/s40593-024-00406-0_bib19","doi-asserted-by":"crossref","first-page":"163","DOI":"10.1016\/j.jpdc.2016.12.028","article-title":"Sauce: A web application for interactive teaching and learning of parallel programming","volume":"105","author":"Hundt","year":"2017","journal-title":"Journal of Parallel and Distributed Computing."},{"key":"10.1007\/s40593-024-00406-0_bib20","doi-asserted-by":"crossref","first-page":"86","DOI":"10.1145\/1930464.1930480","article-title":"Review of recent systems for automatic assessment of programming assignments","author":"Ihantola","year":"2010","journal-title":"Proceedings of the 10th Koli Calling International Conference on Computing Education Research"},{"key":"10.1007\/s40593-024-00406-0_bib21","first-page":"53","article-title":"On the difficulty of learning to program","volume":"4","author":"Jenkins","year":"2002","journal-title":"Proceedings of the 3rd Annual Conference of the LTSN Centre for Information and Computer Sciences"},{"key":"10.1007\/s40593-024-00406-0_bib22","article-title":"Insta-reviewer: A data-driven approach for generating instant feedback on students\u2019 project reports","author":"Jia","year":"2022","journal-title":"International Educational Data Mining Society."},{"issue":"1","key":"10.1007\/s40593-024-00406-0_bib23","first-page":"1","article-title":"A systematic literature review of automated feedback generation for programming exercises","volume":"19","author":"Keuning","year":"2018","journal-title":"ACM Transactions on Computing Education (TOCE)."},{"issue":"2\u20135","key":"10.1007\/s40593-024-00406-0_bib24","first-page":"109","article-title":"A review of techniques in automatic programming assessment for practical skill test","volume":"10","author":"Lajis","year":"2018","journal-title":"Journal of Telecommunication, Electronic and Computer Engineering"},{"issue":"3","key":"10.1007\/s40593-024-00406-0_bib25","doi-asserted-by":"crossref","first-page":"293","DOI":"10.1016\/j.scico.2005.03.007","article-title":"The java. util. concurrent synchronizer framework","volume":"58","author":"Lea","year":"2005","journal-title":"Science of Computer Programming."},{"key":"10.1007\/s40593-024-00406-0_bib26","doi-asserted-by":"crossref","first-page":"186","DOI":"10.1007\/s40593-020-00235-x","article-title":"Natural language generation using deep learning to support mooc learners","volume":"31","author":"Li","year":"2021","journal-title":"International Journal of Artificial Intelligence in Education"},{"key":"10.1007\/s40593-024-00406-0_bib27","doi-asserted-by":"crossref","first-page":"329","DOI":"10.1145\/1346281.1346323","article-title":"Learning from mistakes: a comprehensive study on real world concurrency bug characteristics","author":"Lu","year":"2008","journal-title":"Proceedings of the 13th International Conference on Architectural Support for Programming Languages and Operating Systems"},{"key":"10.1007\/s40593-024-00406-0_bib28","first-page":"37","article-title":"Generating diverse code explanations using the gpt-3 large language model","volume":"2","author":"MacNeil","year":"2022","journal-title":"Proceedings of the 2022 ACM Conference on International Computing Education Research"},{"key":"10.1007\/s40593-024-00406-0_bib29","doi-asserted-by":"crossref","first-page":"194","DOI":"10.1145\/3372782.3406264","article-title":"Adaptive immediate feedback can improve novice programming engagement and intention to persist in computer science","author":"Marwan","year":"2020","journal-title":"Proceedings of the 2020 ACM Conference on International Computing Education Research"},{"key":"10.1007\/s40593-024-00406-0_bib30","series-title":"Proceedings of the 6th International Workshop on Automating Test Case Design, Selection and Evaluation. A-TEST 2015","first-page":"31","article-title":"Concurrent software testing in practice: a catalog of tools","author":"Melo","year":"2015"},{"issue":"1","key":"10.1007\/s40593-024-00406-0_bib31","doi-asserted-by":"crossref","DOI":"10.1145\/3636515","article-title":"Automated grading and feedback tools for programming education: A systematic review","volume":"24","author":"Messer","year":"2024","journal-title":"ACM Trans. Comput. Educ."},{"issue":"2","key":"10.1007\/s40593-024-00406-0_bib32","doi-asserted-by":"crossref","first-page":"227","DOI":"10.1086\/461491","article-title":"Strategies for correcting students\u2019 wrong responses","volume":"87","author":"Meyer","year":"1986","journal-title":"The Elementary School Journal"},{"issue":"2","key":"10.1007\/s40593-024-00406-0_bib33","doi-asserted-by":"crossref","first-page":"100050","DOI":"10.1016\/j.rmal.2023.100050","article-title":"Exploring the potential of using an ai language model for automated essay scoring","volume":"2","author":"Mizumoto","year":"2023","journal-title":"Research Methods in Applied Linguistics."},{"issue":"1","key":"10.1007\/s40593-024-00406-0_bib34","doi-asserted-by":"crossref","first-page":"74","DOI":"10.1145\/130616.130623","article-title":"What are race conditions? some issues and formalizations","volume":"1","author":"Netzer","year":"1992","journal-title":"ACM Letters on Programming Languages and Systems (LOPLAS)."},{"issue":"2","key":"10.1007\/s40593-024-00406-0_bib35","doi-asserted-by":"crossref","first-page":"199","DOI":"10.1080\/03075070600572090","article-title":"Formative assessment and self-regulated learning: A model and seven principles of good feedback practice","volume":"31","author":"Nicol","year":"2006","journal-title":"Studies in Higher Education"},{"issue":"3","key":"10.1007\/s40593-024-00406-0_bib36","article-title":"Automated assessment in computer science education: A state-of-the-art review","volume":"22","author":"Paiva","year":"2022","journal-title":"ACM Transactions on Computing Education (TOCE)"},{"key":"10.1007\/s40593-024-00406-0_bib37","doi-asserted-by":"crossref","first-page":"59","DOI":"10.1016\/j.jss.2015.04.064","article-title":"A large-scale study on the usage of java\u2019s concurrent programming constructs","volume":"106","author":"Pinto","year":"2015","journal-title":"Journal of Systems and Software"},{"key":"10.1007\/s40593-024-00406-0_bib38","series-title":"Proceedings of the 22nd IEEE\/ACM International Conference on Automated Software Engineering. ASE \u201907","first-page":"513","article-title":"Unit testing concurrent software","author":"Pugh","year":"2007"},{"key":"10.1007\/s40593-024-00406-0_bib39","unstructured":"Radford, A., Narasimhan, K., Salimans, T., Sutskever, I., et al (2018). Improving language understanding by generative pre-training."},{"key":"10.1007\/s40593-024-00406-0_bib40","series-title":"Proceedings of the Twenty Third International Conference on Artificial Intelligence and Statistics. Proceedings of Machine Learning Research","first-page":"2435","article-title":"How fine can fine-tuning be? learning efficient language models","author":"Radiya-Dixit","year":"2020"},{"key":"10.1007\/s40593-024-00406-0_bib41","unstructured":"Rahman, K.A., Nordin, M.J. (2007). A review on the static analysis approach in the automated programming assessment systems."},{"key":"10.1007\/s40593-024-00406-0_bib42","doi-asserted-by":"crossref","first-page":"119","DOI":"10.1007\/BF00117714","article-title":"Formative assessment and the design of instructional systems","volume":"18","author":"Sadler","year":"1989","journal-title":"Instructional Science"},{"key":"10.1007\/s40593-024-00406-0_bib43","first-page":"27","article-title":"Automatic generation of programming exercises and code explanations using large language models","volume":"1","author":"Sarsa","year":"2022","journal-title":"Proceedings of the 2022 ACM Conference on International Computing Education Research"},{"key":"10.1007\/s40593-024-00406-0_bib44","unstructured":"Schaeffer, R., Miranda, B., Koyejo, S. (2023). Are emergent abilities of large language models a mirage? arXiv:2304.15004"},{"key":"10.1007\/s40593-024-00406-0_bib45","series-title":"Integrated Formal Methods: 12th International Conference, IFM 2016, Reykjavik, Iceland, June 1-5, 2016, Proceedings 12","first-page":"193","article-title":"Towards a thread-local proof technique for starvation freedom","author":"Schellhorn","year":"2016"},{"issue":"1","key":"10.1007\/s40593-024-00406-0_bib46","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/s40594-019-0192-1","article-title":"Developing student 21st century skills in selected exemplary inclusive stem high schools","volume":"6","author":"Stehle","year":"2019","journal-title":"International Journal of STEM education."},{"issue":"3","key":"10.1007\/s40593-024-00406-0_bib47","first-page":"202","article-title":"The free lunch is over: A fundamental turn toward concurrency in software","volume":"30","author":"Sutter","year":"2005","journal-title":"Dr. Dobb\u2019s Journal."},{"key":"10.1007\/s40593-024-00406-0_bib48","series-title":"2022 2nd International Mobile, Intelligent, and Ubiquitous Computing Conference (MIUCC)","first-page":"230","article-title":"Review of programming assignments automated assessment systems.","author":"Tarek","year":"2022"},{"key":"10.1007\/s40593-024-00406-0_bib49","article-title":"Attention is all you need","volume":"30","author":"Vaswani","year":"2017","journal-title":"Advances in neural information processing systems."},{"key":"10.1007\/s40593-024-00406-0_bib50","unstructured":"Wei, J., Tay, Y., Bommasani, R., Raffel, C., Zoph, B., Borgeaud, S., Yogatama, D., Bosma, M., Zhou, D., Metzler, D., et al (2022). Emergent abilities of large language models. arXiv:2206.07682"},{"key":"10.1007\/s40593-024-00406-0_bib51","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/j.infsof.2016.04.004","article-title":"An extensive empirical study on c++ concurrency constructs","volume":"76","author":"Wu","year":"2016","journal-title":"Information and Software Technology"},{"key":"10.1007\/s40593-024-00406-0_bib52","article-title":"The effect of generative artificial intelligence (ai)-based tool use on students\u2019 computational thinking skills, programming self-efficacy and motivation","volume":"4","author":"Yilmaz","year":"2023","journal-title":"Computers and Education: Artificial Intelligence."},{"key":"10.1007\/s40593-024-00406-0_bib53","series-title":"2017 24th Asia-Pacific Software Engineering Conference (APSEC)","first-page":"594","article-title":"How do programmers maintain concurrent code?","author":"Yu","year":"2017"},{"key":"10.1007\/s40593-024-00406-0_bib54","doi-asserted-by":"crossref","first-page":"23250","DOI":"10.1609\/aaai.v38i21.30372","article-title":"Students\u2019 perceptions and preferences of generative artificial intelligence feedback for programming","volume":"38","author":"Zhang","year":"2024","journal-title":"Proceedings of the AAAI Conference on Artificial Intelligence"},{"issue":"1","key":"10.1007\/s40593-024-00406-0_bib55","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1207\/s15326985ep2501_2","article-title":"Self-regulated learning and academic achievement: An overview","volume":"25","author":"Zimmerman","year":"1990","journal-title":"Educational Psychologist."}],"container-title":["International Journal of Artificial Intelligence in Education"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s40593-024-00406-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s40593-024-00406-0\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1560429226001149?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1560429226001149?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s40593-024-00406-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,18]],"date-time":"2026-05-18T06:22:28Z","timestamp":1779085348000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S1560429226001149"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6]]},"references-count":55,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2025,6]]}},"alternative-id":["S1560429226001149"],"URL":"https:\/\/doi.org\/10.1007\/s40593-024-00406-0","relation":{},"ISSN":["1560-4292"],"issn-type":[{"value":"1560-4292","type":"print"}],"subject":[],"published":{"date-parts":[[2025,6]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Evaluation of LLM Tools for Feedback Generation in a Course on Concurrent Programming","name":"articletitle","label":"Article Title"},{"value":"International Journal of Artificial Intelligence in Education","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1007\/s40593-024-00406-0","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"Copyright \u00a9 2024 The Author(s). Published by Elsevier Ltd","name":"copyright","label":"Copyright"}]}}