{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,27]],"date-time":"2025-03-27T01:04:45Z","timestamp":1743037485863,"version":"3.40.3"},"publisher-location":"Cham","reference-count":32,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783031116438"},{"type":"electronic","value":"9783031116445"}],"license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022]]},"DOI":"10.1007\/978-3-031-11644-5_36","type":"book-chapter","created":{"date-parts":[[2022,7,26]],"date-time":"2022-07-26T14:18:49Z","timestamp":1658845129000},"page":"440-452","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Student-Tutor Mixed-Initiative Decision-Making Supported by\u00a0Deep Reinforcement Learning"],"prefix":"10.1007","author":[{"given":"Song","family":"Ju","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xi","family":"Yang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Tiffany","family":"Barnes","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Min","family":"Chi","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2022,7,27]]},"reference":[{"key":"36_CR1","doi-asserted-by":"crossref","unstructured":"Aleven, V., Koedinger, K.R.: Limitations of student control: do students know when they need help? In: Intelligent Tutoring Systems, pp. 292\u2013303 (2000)","DOI":"10.1007\/3-540-45108-0_33"},{"key":"36_CR2","doi-asserted-by":"crossref","unstructured":"Ausin, M.S., Maniktala, M., Barnes, T., Chi, M.: Tackling the credit assignment problem in reinforcement learning-induced pedagogical policies with neural networks. In: AIED (2021)","DOI":"10.1007\/978-3-030-78292-4_29"},{"issue":"1\u20132","key":"36_CR3","doi-asserted-by":"publisher","first-page":"41","DOI":"10.1023\/A:1022140919877","volume":"13","author":"AG Barto","year":"2003","unstructured":"Barto, A.G., Mahadevan, S.: Recent advances in hierarchical reinforcement learning. Discrete Event Dyn. Syst. 13(1\u20132), 41\u201377 (2003)","journal-title":"Discrete Event Dyn. Syst."},{"key":"36_CR4","doi-asserted-by":"crossref","unstructured":"Cordova, D.I., Lepper, M.R.: Intrinsic motivation and the process of learning: Beneficial effects of contextualization, personalization, and choice. J. Educ. Psychol. 88(4), 715 (1996)","DOI":"10.1037\/\/0022-0663.88.4.715"},{"key":"36_CR5","doi-asserted-by":"crossref","unstructured":"Fachantidis, A., Taylor, M.E., Vlahavas, I.P.: Learning to teach reinforcement learning agents. Mach. Learn. Knowl, Extra. (2017)","DOI":"10.3390\/make1010002"},{"issue":"2","key":"36_CR6","doi-asserted-by":"publisher","first-page":"93","DOI":"10.3200\/JEXE.72.2.93-114","volume":"72","author":"T Flowerday","year":"2004","unstructured":"Flowerday, T., Schraw, G., Stevens, J.: The role of choice and interest in reader engagement. J. Exp. Educ. 72(2), 93\u2013114 (2004)","journal-title":"J. Exp. Educ."},{"key":"36_CR7","doi-asserted-by":"crossref","unstructured":"Ju, S., Zhou, G., Abdelshiheed, M., Barnes, T., Chi: M.: Evaluating critical reinforcement learning framework in the field. In: AIED, pp. 215\u2013227 (2021)","DOI":"10.1007\/978-3-030-78292-4_18"},{"key":"36_CR8","unstructured":"Ju, S., Zhou, G., Barnes, T., Chi, M.: Pick the moment: Identifying critical pedagogical decisions using long-short term rewards. In: EDM (2020)"},{"issue":"2","key":"36_CR9","doi-asserted-by":"publisher","first-page":"5","DOI":"10.1007\/BF02298286","volume":"37","author":"MB Kinzie","year":"1989","unstructured":"Kinzie, M.B., Sullivan, H.J.: Continuing motivation, learner control, and CAI. Education Tech. Research Dev. 37(2), 5\u201314 (1989)","journal-title":"Education Tech. Research Dev."},{"key":"36_CR10","unstructured":"Mandel, T., Liu, Y.E., Levine, S., Brunskill, E., Popovic, Z.: Offline policy evaluation across representations with applications to educational games. In: AAMAS, pp. 1077\u20131084 (2014)"},{"issue":"4","key":"36_CR11","doi-asserted-by":"publisher","first-page":"637","DOI":"10.1007\/s40593-020-00213-3","volume":"30","author":"M Maniktala","year":"2020","unstructured":"Maniktala, M., Cody, C., Barnes, T., Chi, M.: Avoiding help avoidance: Using interface design changes to promote unsolicited hint usage in an intelligent tutor. Int. J. Artif. Intell. Educ. 30(4), 637\u2013667 (2020)","journal-title":"Int. J. Artif. Intell. Educ."},{"issue":"5695","key":"36_CR12","doi-asserted-by":"publisher","first-page":"503","DOI":"10.1126\/science.1100907","volume":"306","author":"SM McClure","year":"2004","unstructured":"McClure, S.M., Laibson, D.I., Loewenstein, G., Cohen, J.D.: Separate neural systems value immediate and delayed monetary rewards. Science 306(5695), 503\u2013507 (2004)","journal-title":"Science"},{"key":"36_CR13","unstructured":"Mitrovic, A., Martin, B.: Scaffolding and fading problem selection in sql-tutor. In: AIED, pp. 479\u2013481 (2003)"},{"key":"36_CR14","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih, V., et al.: Human-level control through deep reinforcement learning. Nature 518, 529\u2013533 (2015)","journal-title":"Nature"},{"issue":"8","key":"36_CR15","first-page":"1057","volume":"9","author":"G Morris","year":"2006","unstructured":"Morris, G., Nevet, A., Arkadir, D., Vaadia, E., Bergman, H.: Midbrain dopamine neurons encode decisions for future action. NatureNeuro 9(8), 1057\u20131063 (2006)","journal-title":"NatureNeuro"},{"key":"36_CR16","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"171","DOI":"10.1007\/978-3-319-08786-3_15","volume-title":"User Modeling, Adaptation, and Personalization","author":"AS Najar","year":"2014","unstructured":"Najar, A.S., Mitrovic, A., McLaren, B.M.: Adaptive support versus alternating worked examples and tutored problems: which leads to better learning? In: Dimitrova, V., Kuflik, T., Chin, D., Ricci, F., Dolog, P., Houben, G.-J. (eds.) UMAP 2014. LNCS, vol. 8538, pp. 171\u2013182. Springer, Cham (2014). https:\/\/doi.org\/10.1007\/978-3-319-08786-3_15"},{"issue":"4","key":"36_CR17","doi-asserted-by":"publisher","first-page":"293","DOI":"10.1080\/00220970209599510","volume":"70","author":"A Renkl","year":"2002","unstructured":"Renkl, A., Atkinson, R.K., Maier, U.H., Staley, R.: From example study to problem solving: smooth transitions help learning. J. Exp. Educ. 70(4), 293\u2013315 (2002)","journal-title":"J. Exp. Educ."},{"issue":"12","key":"36_CR18","doi-asserted-by":"publisher","first-page":"1615","DOI":"10.1038\/nn2013","volume":"10","author":"MR Roesch","year":"2007","unstructured":"Roesch, M.R., Calu, D.J., Schoenbaum, G.: Dopamine neurons encode the better option in rats deciding between different delayed or sized rewards. Nat. Neurosci. 10(12), 1615\u20131624 (2007)","journal-title":"Nat. Neurosci."},{"key":"36_CR19","first-page":"169","volume":"2","author":"I Roll","year":"2014","unstructured":"Roll, I., Wiese, E.S., Long, Y., Aleven, V., Koedinger, K.R.: Tutoring self-and co-regulation with intelligent tutoring systems to help students acquire better learning skills. Design Recomm. Intell. Tutoring Syst. 2, 169\u2013182 (2014)","journal-title":"Design Recomm. Intell. Tutoring Syst."},{"key":"36_CR20","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"419","DOI":"10.1007\/978-3-319-19773-9_42","volume-title":"Artificial Intelligence in Education","author":"JP Rowe","year":"2015","unstructured":"Rowe, J.P., Lester, J.C.: Improving student problem solving in narrative-centered learning environments: a modular reinforcement Learning framework. In: Conati, C., Heffernan, N., Mitrovic, A., Verdejo, M.F. (eds.) AIED 2015. LNCS (LNAI), vol. 9112, pp. 419\u2013428. Springer, Cham (2015). https:\/\/doi.org\/10.1007\/978-3-319-19773-9_42"},{"key":"36_CR21","doi-asserted-by":"publisher","first-page":"161","DOI":"10.1016\/j.learninstruc.2018.06.006","volume":"58","author":"S Schneider","year":"2018","unstructured":"Schneider, S., Nebel, S., Beege, M., Rey, G.D.: The autonomy-enhancing effects of choice on cognitive load, motivation and learning with digital media. Learn. Instr. 58, 161\u2013172 (2018)","journal-title":"Learn. Instr."},{"key":"36_CR22","doi-asserted-by":"crossref","unstructured":"Shen, S., Chi, M.: Reinforcement learning: the sooner the better, or the later the better? In: UMAP, pp. 37\u201344. ACM (2016)","DOI":"10.1145\/2930238.2930247"},{"issue":"9","key":"36_CR23","doi-asserted-by":"publisher","first-page":"1202","DOI":"10.1038\/nn.2881","volume":"14","author":"JH Sul","year":"2011","unstructured":"Sul, J.H., Jo, S., Lee, D., Jung, M.W.: Role of rodent secondary motor cortex in value-based action selection. Nat. Neurosci. 14(9), 1202\u20131208 (2011)","journal-title":"Nat. Neurosci."},{"key":"36_CR24","unstructured":"Sutton, R., Barto, A.: Reinforcement Learning: An Introduction. MIT Press (2018)"},{"key":"36_CR25","unstructured":"Torrey, L., Taylor, M.E.: Teaching on a budget: agents advising agents in reinforcement learning. In: AAMAS, pp. 1053\u20131060 (2013)"},{"issue":"3","key":"36_CR26","doi-asserted-by":"publisher","first-page":"212","DOI":"10.1016\/j.cedpsych.2010.10.004","volume":"36","author":"T Van Gog","year":"2011","unstructured":"Van Gog, T., Kester, L., Paas, F.: Effects of worked examples, example-problem, and problem-example pairs on novices\u2019 learning. Contemp. Educ. Psychol. 36(3), 212\u2013218 (2011)","journal-title":"Contemp. Educ. Psychol."},{"key":"36_CR27","doi-asserted-by":"crossref","unstructured":"Wang, P., Rowe, J., Min, W., Mott, B., Lester, J.: Interactive narrative personalization with deep reinforcement learning. In: IJCAI (2017)","DOI":"10.24963\/ijcai.2017\/538"},{"issue":"2","key":"36_CR28","doi-asserted-by":"publisher","first-page":"153","DOI":"10.1016\/S0360-1315(99)00030-5","volume":"33","author":"H Wood","year":"1999","unstructured":"Wood, H., Wood, D.: Help seeking, learning and contingent tutoring. Comput. Educ. 33(2), 153\u2013169 (1999)","journal-title":"Comput. Educ."},{"key":"36_CR29","doi-asserted-by":"crossref","unstructured":"Zhou, G., Azizsoltani, H., Ausin, M.S., Barnes, T., Chi, M.: Hierarchical reinforcement learning for pedagogical policy induction. In: AIED, pp. 544\u2013556 (2019)","DOI":"10.1007\/978-3-030-23204-7_45"},{"key":"36_CR30","unstructured":"Zhou, G., Chi, M.: The impact of decision agency & granularity on aptitude treatment interaction in tutoring. In: CogSci, pp. 3652\u20133657 (2017)"},{"key":"36_CR31","doi-asserted-by":"crossref","unstructured":"Zhou, G., Yang, X., Azizsoltani, H., Barnes, T., Chi, M.: Improving student-tutor interaction through data-driven explanation of hierarchical reinforcement induced pedagogical policies. In: UMAP. ACM (2020)","DOI":"10.1145\/3340631.3394848"},{"key":"36_CR32","unstructured":"Zimmer, M., Viappiani, P., Weng, P.: Teacher-student framework: A reinforcement learning approach. In: AAMAS Workshop (2013)"}],"container-title":["Lecture Notes in Computer Science","Artificial Intelligence in Education"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-11644-5_36","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,13]],"date-time":"2024-03-13T19:52:55Z","timestamp":1710359575000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-11644-5_36"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"ISBN":["9783031116438","9783031116445"],"references-count":32,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-11644-5_36","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2022]]},"assertion":[{"value":"27 July 2022","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"AIED","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Artificial Intelligence in Education","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Durham","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"United Kingdom","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27 July 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"31 July 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"23","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"aied2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/aied2022.webspace.durham.ac.uk\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"243","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"40","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"40","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"16% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"5","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}