{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,25]],"date-time":"2026-04-25T21:28:30Z","timestamp":1777152510882,"version":"3.51.4"},"publisher-location":"Cham","reference-count":33,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030782917","type":"print"},{"value":"9783030782924","type":"electronic"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-78292-4_10","type":"book-chapter","created":{"date-parts":[[2021,6,10]],"date-time":"2021-06-10T21:03:59Z","timestamp":1623359039000},"page":"113-124","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":9,"title":["Adaptively Scaffolding Cognitive Engagement with Batch Constrained Deep Q-Networks"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4802-3979","authenticated-orcid":false,"given":"Fahmid Morshed","family":"Fahid","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2038-9239","authenticated-orcid":false,"given":"Jonathan P.","family":"Rowe","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Randall D.","family":"Spain","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1396-9887","authenticated-orcid":false,"given":"Benjamin S.","family":"Goldberg","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Robert","family":"Pokorny","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"James","family":"Lester","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2021,6,11]]},"reference":[{"key":"10_CR1","unstructured":"Ai, F., Chen, Y., Guo, Y., Zhao, Y., Wang, Z., Fu, G.: Concept-aware deep knowledge tracing and exercise recommendation in an online learning system. In: Proceedings of the 12th International Conference on Educational Data Mining, pp. 240\u2013245 (2019)"},{"key":"10_CR2","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"472","DOI":"10.1007\/978-3-030-52237-7_38","volume-title":"Artificial Intelligence in Education","author":"M Sanz Ausin","year":"2020","unstructured":"Sanz Ausin, M., Maniktala, M., Barnes, T., Chi, M.: Exploring the impact of simple explanations and agency on batch deep reinforcement learning induced pedagogical policies. In: Bittencourt, I.I., Cukurova, M., Muldner, K., Luckin, R., Mill\u00e1n, E. (eds.) AIED 2020. LNCS (LNAI), vol. 12163, pp. 472\u2013485. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-52237-7_38"},{"key":"10_CR3","unstructured":"Ausin, M.S., Azizsoltani, H., Barnes, T., Chi, M.: Leveraging deep reinforcement learning for pedagogical policy induction in an intelligent tutoring system. In: Proceedings of the 12th International Conference on Educational Data Mining, pp. 168\u2013177 (2019)"},{"key":"10_CR4","doi-asserted-by":"publisher","unstructured":"Azizsoltani, H., Jin, Y.: Unobserved is not equal to non-existent: using Gaussian processes to infer immediate rewards across contexts. In: Proceedings of the 28th International Joint Conference on Artificial Intelligence, pp. 1974\u20131980 (2019). https:\/\/doi.org\/10.24963\/ijcai.2019\/273","DOI":"10.24963\/ijcai.2019\/273"},{"key":"10_CR5","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"224","DOI":"10.1007\/978-3-642-13388-6_27","volume-title":"Intelligent Tutoring Systems","author":"M Chi","year":"2010","unstructured":"Chi, M., VanLehn, K., Litman, D.: Do micro-level tutorial decisions matter: applying reinforcement learning to induce pedagogical tutorial tactics. In: Aleven, V., Kay, J., Mostow, J. (eds.) ITS 2010. LNCS, vol. 6094, pp. 224\u2013234. Springer, Heidelberg (2010). https:\/\/doi.org\/10.1007\/978-3-642-13388-6_27"},{"issue":"6","key":"10_CR6","doi-asserted-by":"publisher","first-page":"1777","DOI":"10.1111\/cogs.12626","volume":"42","author":"MTH Chi","year":"2018","unstructured":"Chi, M.T.H., et al.: Translating the ICAP theory of cognitive engagement into practice. Cogn. Sci. 42(6), 1777\u20131832 (2018). https:\/\/doi.org\/10.1111\/cogs.12626","journal-title":"Cogn. Sci."},{"issue":"4","key":"10_CR7","doi-asserted-by":"publisher","first-page":"219","DOI":"10.1080\/00461520.2014.965823","volume":"49","author":"MTH Chi","year":"2014","unstructured":"Chi, M.T.H., Wylie, R.: The ICAP framework: Linking cognitive engagement to active learning outcomes. Educ. Psychol. 49(4), 219\u2013243 (2014). https:\/\/doi.org\/10.1080\/00461520.2014.965823","journal-title":"Educ. Psychol."},{"issue":"4","key":"10_CR8","doi-asserted-by":"publisher","first-page":"568","DOI":"10.1007\/s40593-019-00187-x","volume":"29","author":"S Doroudi","year":"2019","unstructured":"Doroudi, S., Aleven, V., Brunskill, E.: Where\u2019s the reward? Int. J. Artif. Intell. Educ. 29(4), 568\u2013620 (2019). https:\/\/doi.org\/10.1007\/s40593-019-00187-x","journal-title":"Int. J. Artif. Intell. Educ."},{"key":"10_CR9","unstructured":"Fujimoto, S., Meger, D., Precup, D.: Off-policy deep reinforcement learning without exploration. In: Proceedings of the 36th International Conference on Machine Learning, pp. 2052\u20132062 (2019)"},{"key":"10_CR10","unstructured":"Georgila, K., Core, M.G., Nye, B.D., Karumbaiah, S., Auerbach, D., Ram, M.: Using reinforcement learning to optimize the policies of an intelligent tutoring system for interpersonal skills training. In: Proceedings of the 18th International Conference on Autonomous Agents and Multiagent Systems, pp. 737\u2013745. IFAAMAS, Richland (2019). https:\/\/dl.acm.org\/doi\/abs\/10.5555\/3306127.3331763"},{"key":"10_CR11","doi-asserted-by":"crossref","unstructured":"Van Hasselt, H., Guez, A., Silver, D.: Deep reinforcement learning with double q-learning. In: Proceedings of the 30th AAAI Conference on Artificial Intelligence, pp. 2094\u20132100 (2016)","DOI":"10.1609\/aaai.v30i1.10295"},{"key":"10_CR12","doi-asserted-by":"crossref","unstructured":"Hessel, M., et al.: Rainbow: combining improvements in deep reinforcement learning. In: Proceedings of the 32nd AAAI Conference on Artificial Intelligence, pp. 3215\u20133222 (2018)","DOI":"10.1609\/aaai.v32i1.11796"},{"issue":"8","key":"10_CR13","doi-asserted-by":"publisher","first-page":"1735","DOI":"10.1162\/neco.1997.9.8.1735","volume":"9","author":"S Hochreiter","year":"1997","unstructured":"Hochreiter, S., Schmidhuber, J.: Long short-term memory. Neural Comput. 9(8), 1735\u20131780 (1997). https:\/\/doi.org\/10.1162\/neco.1997.9.8.1735","journal-title":"Neural Comput."},{"key":"10_CR14","unstructured":"Jiang, N., Li, L.: Doubly robust off-policy value evaluation for reinforcement learning. In: Proceedings of the 33rd International Conference on Machine Learning, pp. 652\u2013661 (2016)"},{"key":"10_CR15","unstructured":"Ju, S., Zhou, G., Barnes, T., Chi, M.: Pick the moment: identifying critical pedagogical decisions using long-short term rewards. In: Proceedings of the 13th International Conference on Educational Data Mining, pp. 126\u2013136 (2020)"},{"key":"10_CR16","unstructured":"Kingma, D.P., Ba, J.: Adam: a method for stochastic optimization. arXiv preprint arXiv:1412.6980 (2014)"},{"issue":"6","key":"10_CR17","first-page":"90","volume":"1","author":"TM Kodinariya","year":"2013","unstructured":"Kodinariya, T.M., Makwana, P.R.: Review on determining number of cluster in K-Means clustering. Int. J. Adv. Res. Comput. Sci. Manag. Stud. 1(6), 90\u201395 (2013)","journal-title":"Int. J. Adv. Res. Comput. Sci. Manag. Stud."},{"key":"10_CR18","doi-asserted-by":"publisher","unstructured":"Lim, J., et al.: Active learning through discussion: ICAP framework for education in health professions. BMC Med. Educ. 19(1), Article 47 (2019). https:\/\/doi.org\/10.1186\/s12909-019-1901-7","DOI":"10.1186\/s12909-019-1901-7"},{"issue":"1","key":"10_CR19","doi-asserted-by":"publisher","first-page":"87","DOI":"10.1119\/1.2372468","volume":"75","author":"JD Marx","year":"2007","unstructured":"Marx, J.D., Cummings, K.: Normalized change. Am. J. Phys. 75(1), 87\u201391 (2007). https:\/\/doi.org\/10.1119\/1.2372468","journal-title":"Am. J. Phys."},{"key":"10_CR20","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"320","DOI":"10.1007\/978-3-030-23204-7_27","volume-title":"Artificial Intelligence in Education","author":"A Mitrovic","year":"2019","unstructured":"Mitrovic, A., Gordon, M., Piotrkowicz, A., Dimitrova, V.: Investigating the effect of adding nudges to increase engagement in active video watching. In: Isotani, S., Mill\u00e1n, E., Ogan, A., Hastings, P., McLaren, B., Luckin, R. (eds.) AIED 2019. LNCS (LNAI), vol. 11625, pp. 320\u2013332. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-23204-7_27"},{"issue":"7540","key":"10_CR21","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih, V., et al.: Human-level control through deep reinforcement learning. Nature 518(7540), 529\u2013533 (2015). https:\/\/doi.org\/10.1038\/nature14236","journal-title":"Nature"},{"issue":"5","key":"10_CR22","doi-asserted-by":"publisher","first-page":"615","DOI":"10.1007\/s11251-015-9351-z","volume":"43","author":"J van de Pol","year":"2015","unstructured":"van de Pol, J., Volman, M., Oort, F., Beishuizen, J.: The effects of scaffolding in the classroom: support contingency and student independent working time in relation to student achievement, task effort and appreciation of support. Instr. Sci. 43(5), 615\u2013641 (2015). https:\/\/doi.org\/10.1007\/s11251-015-9351-z","journal-title":"Instr. Sci."},{"key":"10_CR23","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"323","DOI":"10.1007\/978-3-319-61425-0_27","volume-title":"Artificial Intelligence in Education","author":"R Sawyer","year":"2017","unstructured":"Sawyer, R., Rowe, J., Lester, J.: Balancing learning and engagement in game-based learning environments with multi-objective reinforcement learning. In: Andr\u00e9, E., Baker, R., Hu, X., Rodrigo, M.M.T., du Boulay, B. (eds.) AIED 2017. LNCS (LNAI), vol. 10331, pp. 323\u2013334. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-61425-0_27"},{"key":"10_CR24","unstructured":"Schaul, T., Quan, J., Antonoglou, I., Silver, D.: Prioritized experience replay. arXiv preprint arXiv:1511.05952 (2015)"},{"key":"10_CR25","unstructured":"Sottilare, R.A., Brawner, K.W., Goldberg, B.S., Holden, H.K.: The generalized intelligent framework for tutoring (GIFT). US Army Research Laboratory\u2013Human Research & Engineering Directorate (ARL-HRED), Orlando (2012)"},{"key":"10_CR26","unstructured":"Spain, R., Rowe, J., Goldberg, B., Pokorny, R., Lester, J.: Enhancing learning outcomes through adaptive remediation with GIFT. In: Proceedings of the Interservice\/Industry Training, Simulation and Education Conference. Paper No. 19275 (2019)"},{"key":"10_CR27","volume-title":"Reinforcement Learning: An Introduction","author":"RS Sutton","year":"2018","unstructured":"Sutton, R.S., Barto, A.G.: Reinforcement Learning: An Introduction, 2nd edn. MIT Press, Cambridge (2018)","edition":"2"},{"key":"10_CR28","unstructured":"Thomas, P., Brunskill, E.: Data-efficient off-policy policy evaluation for reinforcement learning. In: Proceedings of the 33rd International Conference on Machine Learning, pp. 2139\u20132148 (2016)"},{"issue":"8","key":"10_CR29","doi-asserted-by":"publisher","first-page":"553","DOI":"10.18178\/ijiet.2018.8.8.1098","volume":"8","author":"F Wang","year":"2018","unstructured":"Wang, F.: Reinforcement learning in a POMDP based intelligent tutoring system for optimizing teaching strategies. Int. J. Inf. Educ. Technol. 8(8), 553\u2013558 (2018). https:\/\/doi.org\/10.18178\/ijiet.2018.8.8.1098","journal-title":"Int. J. Inf. Educ. Technol."},{"key":"10_CR30","doi-asserted-by":"publisher","unstructured":"Wang, P., Rowe, J., Min, W., Mott, B., Lester, J.: High-fidelity simulated players for interactive narrative planning. In: Proceedings of the 27th International Joint Conference on Artificial Intelligence, pp. 3884\u20133890 (2018). https:\/\/doi.org\/10.24963\/ijcai.2018\/540","DOI":"10.24963\/ijcai.2018\/540"},{"key":"10_CR31","doi-asserted-by":"publisher","unstructured":"Wang, P., Rowe, J.P., Min, W., Mott, B.W., Lester, J.C.: Interactive narrative personalization with deep reinforcement learning. In: Proceedings of the 26th International Joint Conference on Artificial Intelligence, pp. 3852\u20133858 (2017). https:\/\/doi.org\/10.24963\/ijcai.2017\/538","DOI":"10.24963\/ijcai.2017\/538"},{"issue":"2","key":"10_CR32","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1177\/2332858417708567","volume":"3","author":"BL Wiggins","year":"2017","unstructured":"Wiggins, B.L., Eddy, S.L., Grunspan, D.Z., Crowe, A.J.: The ICAP active learning framework predicts the learning gains observed in intensely active classroom experiences. AERA Open. 3(2), 1\u201314 (2017). https:\/\/doi.org\/10.1177\/2332858417708567","journal-title":"AERA Open."},{"key":"10_CR33","doi-asserted-by":"publisher","unstructured":"Zhou, G., Yang, X., Azizsoltani, H., Barnes, T., Chi, M.: Improving student-system interaction through data-driven explanations of hierarchical reinforcement learning induced pedagogical policies. In: Proceedings of the 28th ACM Conference on User Modeling, Adaptation and Personalization, pp. 284\u2013292. ACM, New York (2020). https:\/\/doi.org\/10.1145\/3340631.3394848","DOI":"10.1145\/3340631.3394848"}],"container-title":["Lecture Notes in Computer Science","Artificial Intelligence in Education"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-78292-4_10","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,12,30]],"date-time":"2022-12-30T17:21:27Z","timestamp":1672420887000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-78292-4_10"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030782917","9783030782924"],"references-count":33,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-78292-4_10","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"11 June 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"AIED","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Artificial Intelligence in Education","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Utrecht","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"The Netherlands","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14 June 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18 June 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"22","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"aied2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/aied2021.science.uu.nl\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"209","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"40","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"76","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"19% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"5","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Due to the COVID-19 pandemic the conference was held online.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}