{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,21]],"date-time":"2026-10-21T16:35:52Z","timestamp":1792600552188,"version":"4.5.0"},"reference-count":48,"publisher":"Springer Science and Business Media LLC","issue":"9","license":[{"start":{"date-parts":[[2021,5,25]],"date-time":"2021-05-25T00:00:00Z","timestamp":1621900800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,5,25]],"date-time":"2021-05-25T00:00:00Z","timestamp":1621900800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/100000006","name":"Office of Naval Research","doi-asserted-by":"publisher","award":["N3979-NV-ONR"],"award-info":[{"award-number":["N3979-NV-ONR"]}],"id":[{"id":"10.13039\/100000006","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2021,9]]},"DOI":"10.1007\/s10994-021-05983-y","type":"journal-article","created":{"date-parts":[[2021,5,25]],"date-time":"2021-05-25T19:02:37Z","timestamp":1621969357000},"page":"2389-2418","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":48,"title":["Bandit algorithms to personalize educational chatbots"],"prefix":"10.1007","volume":"110","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2190-2500","authenticated-orcid":false,"given":"William","family":"Cai","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Josh","family":"Grossman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhiyuan Jerry","family":"Lin","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hao","family":"Sheng","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Johnny Tian-Zheng","family":"Wei","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Joseph Jay","family":"Williams","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sharad","family":"Goel","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,5,25]]},"reference":[{"key":"5983_CR1","unstructured":"Agrawal, S., & Goyal, N. (2013, May). Thompson sampling for contextual bandits with linear payoffs. In International Conference on Machine Learning (pp. 127\u2013135). PMLR."},{"key":"5983_CR2","unstructured":"Al-Rfou, R., Pickett, M., Snaider, J., Sung, Y.-H., Strope, B., & Kurzweil, R. (2016). Conversational contextual cues: The case of personalization and history for response ranking. arXiv preprint arXiv:1606.00372"},{"issue":"2","key":"5983_CR3","doi-asserted-by":"publisher","first-page":"64","DOI":"10.1109\/TLT.2009.22","volume":"2","author":"V Aleven","year":"2009","unstructured":"Aleven, V., McLaren, B. M., & Sewall, J. (2009a). Scaling up programming by demonstration for intelligent tutoring systems development: An open-access web site for middle school mathematics learning. IEEE Transactions on Learning Technologies, 2(2), 64\u201378","journal-title":"IEEE Transactions on Learning Technologies"},{"key":"5983_CR4","unstructured":"Aleven, V., Mclaren,B. M., Sewall, J., & Koedinger, K. R. (2009b). A new paradigm for intelligent tutoring systems: Example-tracing tutors. Technical report"},{"issue":"1","key":"5983_CR5","doi-asserted-by":"publisher","first-page":"224","DOI":"10.1007\/s40593-015-0088-2","volume":"26","author":"V Aleven","year":"2016","unstructured":"Aleven, V., McLaren, B. M., Sewall, J., van Velsen, M., Popescu, O., Demi, S., Ringenberg, M., & Koedinger, K. R. (2016). Example-tracing tutors: Intelligent tutor development for non-programmers. International Journal of Artificial Intelligence in Education, 26(1), 224\u2013269","journal-title":"International Journal of Artificial Intelligence in Education"},{"key":"5983_CR6","unstructured":"Andrews, P., De Boni, M., Manandhar, S., & De, M. (2006) Persuasive argumentation in human computer dialogue. In AAAI spring symposium: Argumentation for consumers of healthcare (pp. 8\u20131)"},{"key":"5983_CR7","unstructured":"Bala, K., Kumar, M., Hulawale, S., & Pandita, S. (2017). Chat-bot for college management system using ai. International Research Journal of Engineering and Technology, 4(11), 2030\u20132033."},{"issue":"1","key":"5983_CR8","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1207\/s15516709cog0101_2","volume":"1","author":"DG Bobrow","year":"1977","unstructured":"Bobrow, D. G., & Winograd, T. (1977). An overview of krl, a knowledge representation language. Cognitive science, 1(1), 3\u201346","journal-title":"Cognitive science"},{"key":"5983_CR9","doi-asserted-by":"crossref","unstructured":"Chi, M., VanLehn, K., Litman, D., & Jordan, P. (2011). Empirically evaluating the application of reinforcement learning to the induction of effective and adaptive pedagogical strategies. User Modeling and User-Adapted Interaction, 21(1), 137\u2013180.","DOI":"10.1007\/s11257-010-9093-1"},{"key":"5983_CR10","doi-asserted-by":"crossref","unstructured":"Chu-Carroll, J., Brown,M. K. (1997) Tracking initiative in collaborative dialogue interactions. In Proceedings of the eighth conference on European chapter of the association for computational linguistics (pp. 262\u2013270). Association for Computational Linguistics","DOI":"10.3115\/979617.979651"},{"key":"5983_CR11","unstructured":"Clement, B., Oudeyer, P.-Y., Roy, D., & Lopes, M. (2015). Multi-armed bandits for intelligent tutoring systems., 7(2), 20\u201348"},{"key":"5983_CR12","doi-asserted-by":"publisher","first-page":"495","DOI":"10.1016\/j.compedu.2013.06.010","volume":"68","author":"SD Craig","year":"2013","unstructured":"Craig, S. D., Hu, X., Graesser, A. C., Bargagliotti, A. E., Sterbinsky, A., Cheney, K. R., Okwumabua, T. & Cheney, S. (2013). The impact of a technology-based mathematics after-school program using ALEKS on student\u2019s knowledge and behaviors. Computers & Education, 68, 495\u2013504","journal-title":"Computers & Education"},{"key":"5983_CR13","doi-asserted-by":"crossref","unstructured":"Davis, D., Hauff, C., & Houben, G.-J. (2018) Evaluating crowdworkers as a proxy for online learners in video-based learning contexts. In Proceedings of the ACM on human-computer interaction (pp. 42:1\u201342:16). ACM","DOI":"10.1145\/3274311"},{"key":"5983_CR14","doi-asserted-by":"crossref","unstructured":"Falmagne, J.-C., Albert, D., Doble, C., Eppstein, D., & Hu,X. (2013) Knowledge spaces: Applications in education. Springer Science & Business Media","DOI":"10.1007\/978-3-642-35329-1"},{"issue":"3","key":"5983_CR15","doi-asserted-by":"publisher","first-page":"243","DOI":"10.1007\/s11257-009-9063-7","volume":"19","author":"M Feng","year":"2009","unstructured":"Feng, M., Heffernan, N., & Koedinger, K. (2009). Addressing the assessment challenge with an online system that tutors as it assesses. User Modeling and User-Adapted Interaction, 19(3), 243\u2013266","journal-title":"User Modeling and User-Adapted Interaction"},{"issue":"2","key":"5983_CR16","doi-asserted-by":"publisher","first-page":"180","DOI":"10.3758\/BF03195563","volume":"36","author":"AC Graesser","year":"2004","unstructured":"Graesser, A. C., Lu, S., Jackson, G. T., Mitchell, H. H., Ventura, M., Olney, A., & Louwerse, M. M. (2004). AutoTutor: A tutor with dialogue in natural language. Behavior Research Methods, Instruments, & Computers, 36(2), 180\u2013192","journal-title":"Behavior Research Methods, Instruments, & Computers"},{"key":"5983_CR17","unstructured":"Graesser, A. C., Penumatsa, P., Ventura, M., Cai, Z., & Hu, X. (2007). Using lsa in autotutor: Learning through mixed initiative dialogue in natural language. Handbook of latent semantic analysis, 243\u2013262"},{"issue":"6","key":"5983_CR18","doi-asserted-by":"publisher","first-page":"495","DOI":"10.1002\/acp.2350090604","volume":"9","author":"AC Graesser","year":"1995","unstructured":"Graesser, A. C., Person, N. K., & Magliano, J. P. (1995). Collaborative dialogue patterns in naturalistic one-to-one tutoring. Applied Cognitive Psychology, 9(6), 495\u2013522","journal-title":"Applied Cognitive Psychology"},{"issue":"4","key":"5983_CR19","first-page":"39","volume":"22","author":"AC Graesser","year":"2001","unstructured":"Graesser, A. C., VanLehn, K., Ros\u00e9, C. P., Jordan, P. W., & Harter, D. (2001). Intelligent tutoring systems with conversational dialogue. AI magazine, 22(4), 39","journal-title":"AI magazine"},{"issue":"1","key":"5983_CR20","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1016\/S1389-0417(99)00005-4","volume":"1","author":"AC Graesser","year":"1999","unstructured":"Graesser, A. C., Wiemer-Hastings, K., Wiemer-Hastings, P., & Kreuz, R. (1999). AutoTutor: A simulation of a human tutor. Cognitive Systems Research, 1(1), 35\u201351","journal-title":"Cognitive Systems Research"},{"issue":"4","key":"5983_CR21","doi-asserted-by":"publisher","first-page":"470","DOI":"10.1007\/s40593-014-0024-x","volume":"24","author":"NT Heffernan","year":"2014","unstructured":"Heffernan, N. T., & Heffernan, C. L. (2014). The assistments ecosystem: Building a platform that brings scientists and teachers together for minimally invasive research on human learning and teaching. International Journal of Artificial Intelligence in Education, 24(4), 470\u2013497","journal-title":"International Journal of Artificial Intelligence in Education"},{"key":"5983_CR22","unstructured":"Horzyk, A., Magierski, S., & Miklaszewski, G. (2009). An intelligent internet shop-assistant recognizing a customer personality for improving man-machine interactions. Recent Advances in intelligent information systems, 13\u201326"},{"key":"5983_CR23","doi-asserted-by":"crossref","unstructured":"Koedinger, K. R., Aleven, V., Heffernan, N., Mclaren, B., & Hockenberry, M. (2004). Opening the door to non-programmers: Authoring intelligent tutor behavior by demonstration. Technical report","DOI":"10.1007\/978-3-540-30139-4_16"},{"issue":"1","key":"5983_CR24","doi-asserted-by":"publisher","first-page":"4","DOI":"10.1016\/0196-8858(85)90002-8","volume":"6","author":"TL Lai","year":"1985","unstructured":"Lai, T. L., & Robbins, H. (1985). Asymptotically efficient adaptive allocation rules. Advances in applied mathematics, 6(1), 4\u201322","journal-title":"Advances in applied mathematics"},{"key":"5983_CR25","unstructured":"Lan, A. S., & Baraniuk, R. G. (2016). A contextual bandits framework for personalized learning action selection. In Proceedings of the 9th international conference on educational data mining (pp. 424\u2013429)"},{"key":"5983_CR26","unstructured":"Lee, S. J., Liu, Y.-E., & Popovic, Z. (2014) Learning individual behavior in an educational game: A data-driven approach. In Proceedings of the 7th international conference on educational data mining (EDM) (pp. 114\u2013121)"},{"key":"5983_CR27","first-page":"661","volume":"2010","author":"L Li","year":"2010","unstructured":"Li, L., Chu, W., Langford, J., & Schapire, R. E. (2010). A contextual-bandit approach to personalized news article recommendation. WWW, 2010, 661\u2013670","journal-title":"WWW"},{"key":"5983_CR28","doi-asserted-by":"crossref","unstructured":"Li, L., Chu, W., Langford, J., & Wang, X. (2011, February). Unbiased offline evaluation of contextual-bandit-based news article recommendation algorithms. In Proceedings of the fourth ACM international conference on Web search and data mining (pp. 297\u2013306).","DOI":"10.1145\/1935826.1935878"},{"issue":"4","key":"5983_CR30","doi-asserted-by":"publisher","first-page":"427","DOI":"10.1007\/s40593-014-0029-5","volume":"24","author":"BD Nye","year":"2014","unstructured":"Nye, B. D., Graesser, A. C., & Hu, X. (2014). AutoTutor and family: A review of 17 years of natural language tutoring. International Journal of Artificial Intelligence in Education, 24(4), 427\u2013469","journal-title":"International Journal of Artificial Intelligence in Education"},{"issue":"1","key":"5983_CR31","doi-asserted-by":"publisher","first-page":"12","DOI":"10.1186\/s40594-018-0109-4","volume":"5","author":"BD Nye","year":"2018","unstructured":"Nye, B. D., Pavlik, P. I., Windsor, A., Olney, A. M., Hajeer, M., & Hu, X. (2018). SKOPE-IT (Shareable Knowledge Objects as Portable Intelligent Tutors): overlaying natural language tutoring on an adaptive learning system for mathematics. International Journal of STEM Education, 5(1), 12","journal-title":"International Journal of STEM Education"},{"key":"5983_CR32","doi-asserted-by":"crossref","unstructured":"O'Rourke, E., Andersen, E., Gulwani, S., & Popovi\u0107, Z. (2015, April). A framework for automatically generating interactive instructional scaffolding. In Proceedings of the 33rd annual ACM conference on human factors in computing systems (pp. 1545\u20131554).","DOI":"10.1145\/2702123.2702580"},{"key":"5983_CR33","first-page":"47","volume":"97","author":"NK Person","year":"2003","unstructured":"Person, N. K. (2003). AutoTutor improves deep learning of computer literacy: Is it the dialog or the talking head. Artificial intelligence in education: Shaping the future of learning through intelligent technologies, 97, 47","journal-title":"Artificial intelligence in education: Shaping the future of learning through intelligent technologies"},{"key":"5983_CR34","unstructured":"Quarteroni, S., & Manandhar, S. (2007). A chatbot-based interactive question answering system. Decalog 2007, 83"},{"key":"5983_CR35","doi-asserted-by":"crossref","unstructured":"Raux, A., & Eskenazi, M. (2009) A finite-state turn-taking model for spoken dialog systems. In Proceedings of human language technologies: The 2009 annual conference of the North American chapter of the association for computational linguistics (pp. 629\u2013637). Association for Computational Linguistics","DOI":"10.3115\/1620754.1620846"},{"issue":"2","key":"5983_CR36","doi-asserted-by":"publisher","first-page":"249","DOI":"10.3758\/BF03194060","volume":"14","author":"S Ritter","year":"2007","unstructured":"Ritter, S., Anderson, J. R., Koedinger, K. R., & Corbett, A. (2007). Cognitive tutor: Applied research in mathematics education. Psychonomic Bulletin & Review, 14(2), 249\u2013255","journal-title":"Psychonomic Bulletin & Review"},{"key":"5983_CR37","doi-asserted-by":"crossref","unstructured":"Ruan, S., Jiang, L., Xu, J., Tham, B. J. K., Qiu, Z., Zhu, Y., \u2026 Landay, J. A. (2019, May). Quizbot: A dialogue-based adaptive learning system for factual knowledge. In Proceedings of the 2019 CHI Conference on Human Factors in Computing Systems (pp. 1\u201313).","DOI":"10.1145\/3290605.3300587"},{"key":"5983_CR38","doi-asserted-by":"crossref","unstructured":"Segal, A., David, Y. B., Williams, J. J., Gal, K., & Shalom, Y. (2018, June). Combining difficulty ranking with multi-armed bandits to sequence educational content. In International conference on artificial intelligence in education (pp. 317\u2013321). Cham: Springer.","DOI":"10.1007\/978-3-319-93846-2_59"},{"issue":"1","key":"5983_CR39","first-page":"61","volume":"18","author":"S Seneff","year":"1992","unstructured":"Seneff, S. (1992). Tina: A natural language system for spoken language applications. Computational linguistics, 18(1), 61\u201386","journal-title":"Computational linguistics"},{"key":"5983_CR40","doi-asserted-by":"crossref","unstructured":"Shin, H., Ko, E.-Y., Williams, J. J., & Kim, J. (2018). Understanding the effect of in-video prompting on learners and instructors. In Proceedings of the 2018 CHI Conference on Human Factors in Computing Systems (p. 319). ACM","DOI":"10.1145\/3173574.3173893"},{"key":"5983_CR41","first-page":"13","volume-title":"A series of books in psychology","author":"RE Snow","year":"1989","unstructured":"Snow, R. E. (1989). Aptitude-treatment interaction as a framework for research on individual differences in learning. A series of books in psychology (pp. 13\u201359). Advances in theory and research: Learning and individual differences"},{"key":"5983_CR42","doi-asserted-by":"crossref","unstructured":"VanLehn, K. (1996). Conceptual and meta learning during coached problem solving (pp. 29\u201347)","DOI":"10.1007\/3-540-61327-7_99"},{"key":"5983_CR43","doi-asserted-by":"crossref","unstructured":"VanLehn, K., Jordan, P. W., Ros\u00e9, C. P., Bhembe, D., B\u00f6ttner, M., Gaydos, A., Makatchev, M., Pappuswamy, U., Ringenberg, M., Roque, A., Siler, S., & Srivastava, R. (2002). The architecture of why2-atlas: A coach for qualitative physics essay writing. In International conference on intelligent tutoring systems (pp. 158\u2013167). Springer","DOI":"10.1007\/3-540-47987-2_20"},{"key":"5983_CR44","doi-asserted-by":"crossref","unstructured":"Walker, M., & Whittaker, S. (1990) Mixed initiative in dialogue: An investigation into discourse segmentation. In Proceedings of the 28th annual meeting on association for computational linguistics (pp. 70\u201378). Association for Computational Linguistics","DOI":"10.3115\/981823.981833"},{"issue":"3","key":"5983_CR45","first-page":"93","volume":"14","author":"B Weeraratne","year":"2018","unstructured":"Weeraratne, B., & Chin, B. (2018). Can khan academy e-learning video tutorials improve mathematics achievement in Srilanka? International Journal of Education and Development Using Information and Communication Technology, 14(3), 93\u2013112","journal-title":"International Journal of Education and Development Using Information and Communication Technology"},{"key":"5983_CR46","doi-asserted-by":"crossref","unstructured":"Winkler, R., Hobert, S., Salovaara, A., S\u00f6llner, M., & Leimeister, J. M. (2020). Sara, the lecturer: Improving learning in online education with a scaffolding-based conversational agent. In Proceedings of the 2020 CHI conference on human factors in computing systems (pp. 1\u201314)","DOI":"10.1145\/3313831.3376781"},{"key":"5983_CR47","doi-asserted-by":"crossref","unstructured":"Xu, A., Liu, Z., Guo, Y., Sinha, V., & Akkiraju, R. (2017). A new chatbot for customer service on social media. In Proceedings of the 2017 CHI conference on human factors in computing systems (pp. 3506\u20133510). ACM","DOI":"10.1145\/3025453.3025496"},{"key":"5983_CR48","doi-asserted-by":"crossref","unstructured":"Yan, R., Song, Y., & Wu, H. (2016). Learning to respond with deep neural networks for retrieval-based human-computer conversation system. In Proceedings of the 39th international ACM SIGIR conference on research and development in information retrieval (pp. 55\u201364). ACM","DOI":"10.1145\/2911451.2911542"},{"key":"5983_CR49","unstructured":"Zhou, G., Wang, J., Lynch, C., & Chi, M. (2017). Towards closing the loop: Bridging machine-induced pedagogical policies to learning theories. In Proceedings of the 10th international conference on educational data mining (pp. 112\u2013119)"}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-021-05983-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10994-021-05983-y\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-021-05983-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,5,25]],"date-time":"2022-05-25T00:05:14Z","timestamp":1653437114000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10994-021-05983-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,5,25]]},"references-count":48,"journal-issue":{"issue":"9","published-print":{"date-parts":[[2021,9]]}},"alternative-id":["5983"],"URL":"https:\/\/doi.org\/10.1007\/s10994-021-05983-y","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"value":"0885-6125","type":"print"},{"value":"1573-0565","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,5,25]]},"assertion":[{"value":"15 May 2020","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"18 January 2021","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"12 April 2021","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 May 2021","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}