{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,26]],"date-time":"2026-07-26T02:58:53Z","timestamp":1785034733541,"version":"3.55.0"},"reference-count":38,"publisher":"Springer Science and Business Media LLC","issue":"10","license":[{"start":{"date-parts":[[2024,6,10]],"date-time":"2024-06-10T00:00:00Z","timestamp":1717977600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,6,10]],"date-time":"2024-06-10T00:00:00Z","timestamp":1717977600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Knowl Inf Syst"],"published-print":{"date-parts":[[2024,10]]},"DOI":"10.1007\/s10115-024-02156-w","type":"journal-article","created":{"date-parts":[[2024,6,10]],"date-time":"2024-06-10T19:01:25Z","timestamp":1718046085000},"page":"6133-6150","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":19,"title":["LLM examiner: automating assessment in informal self-directed e-learning using ChatGPT"],"prefix":"10.1007","volume":"66","author":[{"given":"Nursultan","family":"Askarbekuly","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nenad","family":"Ani\u010di\u0107","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,6,10]]},"reference":[{"issue":"2","key":"2156_CR1","doi-asserted-by":"publisher","first-page":"60","DOI":"10.37965\/jait.2023.0184","volume":"3","author":"MA AlAfnan","year":"2023","unstructured":"AlAfnan MA, Dishari S, Jovic M, Lomidze K (2023) ChatGPT as an educational tool: opportunities, challenges, and recommendations for communication, business writing, and composition courses. J Artif Intel Technol 3(2):60\u201368. https:\/\/doi.org\/10.37965\/jait.2023.0184","journal-title":"J Artif Intel Technol"},{"issue":"3","key":"2156_CR2","doi-asserted-by":"publisher","first-page":"278","DOI":"10.1016\/j.aiepr.2023.03.003","volume":"6","author":"S Badini","year":"2023","unstructured":"Badini S, Regondi S, Frontoni E, Pugliese R (2023) Assessing the capabilities of ChatGPT to improve additive manufacturing troubleshooting. Adv Ind Eng Polym Res 6(3):278\u2013287. https:\/\/doi.org\/10.1016\/j.aiepr.2023.03.003","journal-title":"Adv Ind Eng Polym Res"},{"issue":"1","key":"2156_CR3","doi-asserted-by":"publisher","first-page":"380","DOI":"10.33395\/jmp.v12i2.12407","volume":"12","author":"AD Subagja","year":"2023","unstructured":"Subagja AD, Almaududi\u00a0Ausat AM, Sari AR, Wanof MI, Suherlan S (2023) Improving customer service quality in MSMEs through the use of ChatGPT. J Minfo Polgan 12(1):380\u2013386. https:\/\/doi.org\/10.33395\/jmp.v12i2.12407","journal-title":"J Minfo Polgan"},{"issue":"1","key":"2156_CR4","doi-asserted-by":"publisher","first-page":"1","DOI":"10.54489\/ijtim.v3i1.195","volume":"3","author":"A Al-sa\u2019di","year":"2023","unstructured":"Al-sa\u2019di A, Miller D (2023) Exploring the impact of artificial intelligence language model ChatGPT on the user experience. Int J Technol Innov Manag 3(1):1\u20138. https:\/\/doi.org\/10.54489\/ijtim.v3i1.195","journal-title":"Int J Technol Innov Manag"},{"key":"2156_CR5","doi-asserted-by":"publisher","first-page":"121","DOI":"10.1016\/j.iotcps.2023.04.003","volume":"3","author":"PP Ray","year":"2023","unstructured":"Ray PP (2023) ChatGPT: a comprehensive review on background, applications, key challenges, bias, ethics, limitations and future scope. Internet Things Cyber-Phys Syst 3:121\u2013154. https:\/\/doi.org\/10.1016\/j.iotcps.2023.04.003","journal-title":"Internet Things Cyber-Phys Syst"},{"issue":"1","key":"2156_CR6","doi-asserted-by":"publisher","first-page":"39","DOI":"10.1080\/07294360.2012.642839","volume":"31","author":"J Biggs","year":"2012","unstructured":"Biggs J (2012) What the student does: teaching for enhanced learning. High Educ Res Dev 31(1):39\u201355","journal-title":"High Educ Res Dev"},{"key":"2156_CR7","doi-asserted-by":"publisher","first-page":"190","DOI":"10.1016\/j.sbspro.2014.01.1155","volume":"112","author":"K Saks","year":"2014","unstructured":"Saks K, Leijen A (2014) Distinguishing self-directed and self-regulated learning and measuring them in the e-learning context. Proc Soc Behav Sci 112:190\u2013198. https:\/\/doi.org\/10.1016\/j.sbspro.2014.01.1155","journal-title":"Proc Soc Behav Sci"},{"key":"2156_CR8","doi-asserted-by":"publisher","unstructured":"Liu F-J (2008) Design of self-directed e-learning material recommendation system with on-line evaluation. In: 2008 international conference on convergence and hybrid information technology, pp. 274\u2013277. https:\/\/doi.org\/10.1109\/ICHIT.2008.184","DOI":"10.1109\/ICHIT.2008.184"},{"key":"2156_CR9","unstructured":"Union E (2023) Europass Tools: European qualifications framework. https:\/\/europa.eu\/europass\/en\/europass-tools\/european-qualifications-framework. Accessed: May 15, 2023"},{"issue":"3","key":"2156_CR10","doi-asserted-by":"publisher","first-page":"1919","DOI":"10.1016\/j.compedu.2011.04.005","volume":"57","author":"A Keramati","year":"2011","unstructured":"Keramati A, Afshari-Mofrad M, Kamrani A (2011) The role of readiness factors in e-learning outcomes: an empirical study. Comput Educ 57(3):1919\u20131929. https:\/\/doi.org\/10.1016\/j.compedu.2011.04.005","journal-title":"Comput Educ"},{"issue":"2","key":"2156_CR11","doi-asserted-by":"publisher","first-page":"554","DOI":"10.1016\/j.compedu.2010.02.017","volume":"55","author":"M \u0106uku\u0161i\u0107","year":"2010","unstructured":"\u0106uku\u0161i\u0107 M, Alfirevi\u0107 N, Grani\u0107 A (2010) Gara\u010da: e-learning process management and the e-learning performance: results of a European empirical study. Comput Educ 55(2):554\u2013565. https:\/\/doi.org\/10.1016\/j.compedu.2010.02.017","journal-title":"Comput Educ"},{"issue":"6","key":"2156_CR12","first-page":"23","volume":"42","author":"TC Reeves","year":"2002","unstructured":"Reeves TC (2002) Keys to successful e-learning: outcomes, assessment and evaluation. Educ Technol 42(6):23\u201329","journal-title":"Educ Technol"},{"key":"2156_CR13","doi-asserted-by":"crossref","unstructured":"Askarbekuly N, Solovyov A, Lukyanchikova E, Pimenov D, Mazzara M (2021) Building an educational product: constructive alignment and requirements engineering. In: Advances in Artificial Intelligence, Software and Systems Engineering: Proceedings of the AHFE 2021 Virtual Conferences on Human Factors in Software and Systems Engineering, Artificial Intelligence and Social Computing, and Energy, July 25-29, 2021, USA, pp. 358\u2013365. Springer","DOI":"10.1007\/978-3-030-80624-8_44"},{"key":"2156_CR14","doi-asserted-by":"crossref","unstructured":"Askarbekuly N, Sadovykh A, Mazzara M (2020) Combining two modelling approaches: Gqm and kaos in an open source project. In: Open Source Systems: 16th IFIP WG 2.13 International Conference, OSS 2020, Innopolis, Russia, May 12\u201314, 2020, Proceedings 16, pp. 106\u2013119. Springer","DOI":"10.1007\/978-3-030-47240-5_11"},{"issue":"2","key":"2156_CR15","doi-asserted-by":"publisher","first-page":"304","DOI":"10.1080\/02602938.2019.1639613","volume":"45","author":"R Ajjawi","year":"2020","unstructured":"Ajjawi R, Tai J, Huu Nghia TL, Boud D, Johnson L, Patrick C-J (2020) Aligning assessment with the needs of work-integrated learning: the challenges of authentic assessment in a complex context. Assess Eval High Educ 45(2):304\u2013316","journal-title":"Assess Eval High Educ"},{"key":"2156_CR16","doi-asserted-by":"crossref","unstructured":"Kumaran VS, Sankar A (2013) An automated assessment of students\u2019 learning in e-learning using concept map and ontology mapping. In: Advances in Web-Based Learning\u2013ICWL 2013: 12th International Conference, Kenting, Taiwan, October 6\u20139, 2013. Proceedings 12, pp. 274\u2013283. Springer","DOI":"10.1007\/978-3-642-41175-5_28"},{"issue":"1145\/2438653","key":"2156_CR17","first-page":"2438664","volume":"10","author":"JT Folsom-Kovarik","year":"2013","unstructured":"Folsom-Kovarik JT, Sukthankar G, Schatz S (2013) Tractable POMDP representations for intelligent tutoring systems. ACM Trans Intell Syst Technol 10(1145\/2438653):2438664","journal-title":"ACM Trans Intell Syst Technol"},{"issue":"2","key":"2156_CR18","first-page":"260","volume":"33","author":"DS Myers","year":"2017","unstructured":"Myers DS, Chatlani N (2017) Implementing an adaptive tutorial system for coding literacy. J Comput Sci Coll 33(2):260\u2013267","journal-title":"J Comput Sci Coll"},{"issue":"3","key":"2156_CR19","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3513140","volume":"22","author":"JC Paiva","year":"2022","unstructured":"Paiva JC, Leal JP, Figueira \u00c1 (2022) Automated assessment in computer science education: a state-of-the-art review. ACM Trans Comput Educ 22(3):1\u201340","journal-title":"ACM Trans Comput Educ"},{"issue":"3","key":"2156_CR20","first-page":"02","volume":"20","author":"J Crawford","year":"2023","unstructured":"Crawford J, Cowling M, Allen K-A (2023) Leadership is needed for ethical ChatGPT: character, assessment, and learning using artificial intelligence (AI). J Univ Teach Learn Pract 20(3):02","journal-title":"J Univ Teach Learn Pract"},{"key":"2156_CR21","unstructured":"Susnjak T (2022) ChatGPT: the end of online exam integrity? arXiv preprint arXiv:2212.09292"},{"issue":"2","key":"2156_CR22","doi-asserted-by":"publisher","first-page":"421","DOI":"10.30935\/cedtech\/13036","volume":"15","author":"M Halaweh","year":"2023","unstructured":"Halaweh M (2023) ChatGPT in education: strategies for responsible implementation. Contemp Educ Technol 15(2):421. https:\/\/doi.org\/10.30935\/cedtech\/13036","journal-title":"Contemp Educ Technol"},{"issue":"1","key":"2156_CR23","doi-asserted-by":"publisher","first-page":"15","DOI":"10.1186\/s40561-023-00237-x","volume":"10","author":"A Tlili","year":"2023","unstructured":"Tlili A, Shehata B, Adarkwah MA, Bozkurt A, Hickey DT, Huang R, Agyemang B (2023) What if the devil is my guardian angel: ChatGPT as a case study of using chatbots in education. Smart Learn Environ 10(1):15","journal-title":"Smart Learn Environ"},{"issue":"2","key":"2156_CR24","doi-asserted-by":"publisher","first-page":"60","DOI":"10.37965\/jait.2023.0184","volume":"3","author":"MA AlAfnan","year":"2023","unstructured":"AlAfnan MA, Dishari S, Jovic M, Lomidze K (2023) Chatgpt as an educational tool: opportunities, challenges, and recommendations for communication, business writing, and composition courses. J Artif Intel Technol 3(2):60\u201368. https:\/\/doi.org\/10.37965\/jait.2023.0184","journal-title":"J Artif Intel Technol"},{"key":"2156_CR25","doi-asserted-by":"publisher","unstructured":"Firat M (2023) How chat GPT can transform autodidactic experiences and open education? https:\/\/doi.org\/10.31219\/osf.io\/9ge8m","DOI":"10.31219\/osf.io\/9ge8m"},{"issue":"2","key":"2156_CR26","doi-asserted-by":"publisher","first-page":"100115","DOI":"10.1016\/j.tbench.2023.100115","volume":"3","author":"M Javaid","year":"2023","unstructured":"Javaid M, Haleem A, Singh RP, Khan S, Khan IH (2023) Unlocking the opportunities through chatGPT tool towards ameliorating the education system. BenchCouncil Trans Benchmarks Stand Eval 3(2):100115. https:\/\/doi.org\/10.1016\/j.tbench.2023.100115","journal-title":"BenchCouncil Trans Benchmarks Stand Eval"},{"issue":"1","key":"2156_CR27","first-page":"342","volume":"6","author":"J Rudolph","year":"2023","unstructured":"Rudolph J, Tan S, Tan S (2023) ChatGPT: Bullshit spewer or the end of traditional assessments in higher education? J Appl Learn Teach 6(1):342","journal-title":"J Appl Learn Teach"},{"key":"2156_CR28","doi-asserted-by":"publisher","first-page":"106244","DOI":"10.1016\/j.ssci.2023.106244","volume":"167","author":"O Oviedo-Trespalacios","year":"2023","unstructured":"Oviedo-Trespalacios O, Peden AE, Cole-Hunter T, Costantini A, Haghani M, Rod JE, Kelly S, Torkamaan H, Tariq A, David Albert Newton J, Gallagher T, Steinert S, Filtness AJ, Reniers G (2023) The risks of using chatGPT to obtain common safety-related information and advice. Saf Sci 167:106244. https:\/\/doi.org\/10.1016\/j.ssci.2023.106244","journal-title":"Saf Sci"},{"issue":"4","key":"2156_CR29","doi-asserted-by":"publisher","first-page":"1213","DOI":"10.1111\/ijcs.12928","volume":"47","author":"J Paul","year":"2023","unstructured":"Paul J, Ueno A, Dennis C (2023) ChatGPT and consumers: benefits, pitfalls and future research agenda. Int J Consum Stud 47(4):1213\u20131225. https:\/\/doi.org\/10.1111\/ijcs.12928","journal-title":"Int J Consum Stud"},{"key":"2156_CR30","unstructured":"Mei\u00dfner N, Speth S, Kieslinger J, Becker S (2024) Evalquiz\u2013llm-based automated generation of self-assessment quizzes in software engineering education. In: Software Engineering Im Unterricht der Hochschulen 2024, pp. 53\u201364. Gesellschaft f\u00fcr Informatik eV"},{"key":"2156_CR31","doi-asserted-by":"publisher","unstructured":"Kasneci E, Sessler K, K\u00fcchemann S, Bannert M, Dementieva D, Fischer F, Gasser U, Groh G, G\u00fcnnemann S, H\u00fcllermeier E, Krusche S, Kutyniok G, Michaeli T, Nerdel C, Pfeffer J, Poquet O, Sailer M, Schmidt A, Seidel T, Stadler M, Weller J, Kuhn J, Kasneci G (2023) Chatgpt for good? on opportunities and challenges of large language models for education. Learn Individ Differ 103:102274. https:\/\/doi.org\/10.1016\/j.lindif.2023.102274","DOI":"10.1016\/j.lindif.2023.102274"},{"issue":"1","key":"2156_CR32","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1287\/isre.3.1.60","volume":"3","author":"WH DeLone","year":"1992","unstructured":"DeLone WH, McLean ER (1992) Information systems success: the quest for the dependent variable. Inf Syst Res 3(1):60\u201395","journal-title":"Inf Syst Res"},{"issue":"1","key":"2156_CR33","doi-asserted-by":"publisher","first-page":"212","DOI":"10.3390\/jtaer17010011","volume":"17","author":"L Jenneboer","year":"2022","unstructured":"Jenneboer L, Herrando C, Constantinides E (2022) The impact of chatbots on customer loyalty: a systematic literature review. J Theor Appl Electron Commer Res 17(1):212\u2013229. https:\/\/doi.org\/10.3390\/jtaer17010011","journal-title":"J Theor Appl Electron Commer Res"},{"issue":"1","key":"2156_CR34","doi-asserted-by":"publisher","first-page":"46","DOI":"10.1038\/s42256-022-00593-2","volume":"5","author":"A Sharma","year":"2023","unstructured":"Sharma A, Lin IW, Miner AS, Atkins DC, Althoff T (2023) Human\u2013AI collaboration enables more empathic conversations in text-based peer-to-peer mental health support. Nat Mach Intel 5(1):46\u201357. https:\/\/doi.org\/10.1038\/s42256-022-00593-2","journal-title":"Nat Mach Intel"},{"key":"2156_CR35","unstructured":"NamazApp on Apple Store. https:\/\/apps.apple.com\/app\/id1447056625. Accessed: 2024-04-23"},{"key":"2156_CR36","doi-asserted-by":"crossref","unstructured":"Namaz Live. https:\/\/www.namaz.live. Accessed: 2024-04-23","DOI":"10.14267\/TURBULL.2023v23n3.5"},{"key":"2156_CR37","unstructured":"Best Practices for Prompt Engineering with OpenAI API. https:\/\/help.openai.com\/en\/articles\/6654000-best-practices-for-prompt-engineering-with-openai-api. Accessed: August 21, (2023)"},{"key":"2156_CR38","unstructured":"Nurlingo: LLM Examiner. https:\/\/github.com\/nurlingo\/llm-examiner. Accessed: 2024-04-23"}],"container-title":["Knowledge and Information Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-024-02156-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10115-024-02156-w\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-024-02156-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,23]],"date-time":"2024-09-23T05:05:50Z","timestamp":1727067950000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10115-024-02156-w"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,10]]},"references-count":38,"journal-issue":{"issue":"10","published-print":{"date-parts":[[2024,10]]}},"alternative-id":["2156"],"URL":"https:\/\/doi.org\/10.1007\/s10115-024-02156-w","relation":{},"ISSN":["0219-1377","0219-3116"],"issn-type":[{"value":"0219-1377","type":"print"},{"value":"0219-3116","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,6,10]]},"assertion":[{"value":"2 December 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 April 2024","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"26 May 2024","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"10 June 2024","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}