{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,18]],"date-time":"2026-05-18T16:18:37Z","timestamp":1779121117998,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":28,"publisher":"ACM","license":[{"start":{"date-parts":[[2017,4,12]],"date-time":"2017-04-12T00:00:00Z","timestamp":1491955200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100005246","name":"Institute of Education Sciences","doi-asserted-by":"publisher","award":["R305A130215,R305B150008"],"award-info":[{"award-number":["R305A130215,R305B150008"]}],"id":[{"id":"10.13039\/100005246","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2017,4,12]]},"DOI":"10.1145\/3051457.3051463","type":"proceedings-article","created":{"date-parts":[[2017,4,14]],"date-time":"2017-04-14T12:18:54Z","timestamp":1492172334000},"page":"3-12","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":9,"title":["Robust Evaluation Matrix"],"prefix":"10.1145","author":[{"given":"Shayan","family":"Doroudi","sequence":"first","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}]},{"given":"Vincent","family":"Aleven","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}]},{"given":"Emma","family":"Brunskill","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}]}],"member":"320","published-online":{"date-parts":[[2017,4,12]]},"reference":[{"key":"e_1_3_2_1_1_1","first-page":"552","article-title":"ADVISOR: A machine learning architecture for intelligent tutor construction","volume":"2000","author":"Beck Joseph","year":"2000","journal-title":"AAAI\/IAAI"},{"key":"e_1_3_2_1_2_1","volume-title":"Educational Data Mining","author":"Beck Joseph","year":"2013"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11257-010-9093-1"},{"key":"e_1_3_2_1_4_1","unstructured":"Benjamin Clement Pierre-Yves Oudeyer and Manuel Lopes. 2016. A Comparison of Automatic Teaching Strategies for Heterogeneous Student Populations. International Educational Data Mining Society (2016).  Benjamin Clement Pierre-Yves Oudeyer and Manuel Lopes. 2016. A Comparison of Automatic Teaching Strategies for Heterogeneous Student Populations. International Educational Data Mining Society (2016)."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"Albert T Corbett and John R Anderson. 1994. Knowledge tracing: Modeling the acquisition of procedural knowledge. User modeling and user-adapted interaction 4 4 (1994) 253--278.  Albert T Corbett and John R Anderson. 1994. Knowledge tracing: Modeling the acquisition of procedural knowledge. User modeling and user-adapted interaction 4 4 (1994) 253--278.","DOI":"10.1007\/BF01099821"},{"key":"e_1_3_2_1_7_1","unstructured":"Shayan Doroudi Kenneth Holstein Vincent Aleven and Emma Brunskill. 2015. Towards Understanding How to Leverage Sense-Making Induction and Refinement and Fluency to Improve Robust Learning. International Educational Data Mining Society (2015).  Shayan Doroudi Kenneth Holstein Vincent Aleven and Emma Brunskill. 2015. Towards Understanding How to Leverage Sense-Making Induction and Refinement and Fluency to Improve Robust Learning. International Educational Data Mining Society (2015)."},{"key":"e_1_3_2_1_8_1","unstructured":"Miroslav Dud\u00edk John Langford and Lihong Li. 2011. Doubly robust policy evaluation and learning. arXiv preprint arXiv:1103.4601 (2011).  Miroslav Dud\u00edk John Langford and Lihong Li. 2011. Doubly robust policy evaluation and learning. arXiv preprint arXiv:1103.4601 (2011)."},{"key":"e_1_3_2_1_9_1","unstructured":"Jos\u00e9 P Gonz\u00e1lez-Brenes and Yun Huang. 2015. \" Your Model Is Predictive--but Is It Useful?\" Theoretical and Empirical Considerations of a New Paradigm for Adaptive Tutoring Evaluation. International Educational Data Mining Society (2015).  Jos\u00e9 P Gonz\u00e1lez-Brenes and Yun Huang. 2015. \" Your Model Is Predictive--but Is It Useful?\" Theoretical and Empirical Considerations of a New Paradigm for Adaptive Tutoring Evaluation. International Educational Data Mining Society (2015)."},{"key":"e_1_3_2_1_10_1","volume-title":"Proceedings of the 32nd International Conference on Machine Learning (ICML-15)","author":"Hallak Assaf","year":"2015"},{"key":"e_1_3_2_1_11_1","unstructured":"Nan Jiang and Lihong Li. 2015. Doubly Robust Off-policy Evaluation for Reinforcement Learning. arXiv preprint arXiv:1511.03722 (2015).  Nan Jiang and Lihong Li. 2015. Doubly Robust Off-policy Evaluation for Reinforcement Learning. arXiv preprint arXiv:1511.03722 (2015)."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1207\/S15326985EP3801_4"},{"key":"e_1_3_2_1_13_1","unstructured":"Jung In Lee and Emma Brunskill. 2012. The Impact on Individualizing Student Models on Necessary Practice Opportunities. International Educational Data Mining Society (2012).  Jung In Lee and Emma Brunskill. 2012. The Impact on Individualizing Student Models on Necessary Practice Opportunities. International Educational Data Mining Society (2012)."},{"key":"e_1_3_2_1_14_1","unstructured":"Robert V Lindsey Mohammad Khajah and Michael C Mozer. 2014. Automatic discovery of cognitive skills to improve the prediction of student learning. In Advances in neural information processing systems. 1386--1394.  Robert V Lindsey Mohammad Khajah and Michael C Mozer. 2014. Automatic discovery of cognitive skills to improve the prediction of student learning. In Advances in neural information processing systems. 1386--1394."},{"key":"e_1_3_2_1_15_1","volume-title":"Proceedings of the 2014 international conference on Autonomous agents and multi-agent systems. International Foundation for Autonomous Agents and Multiagent Systems, 1077--1084","author":"Mandel Travis","year":"2014"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.1060.0614"},{"key":"e_1_3_2_1_17_1","unstructured":"Christopher M Mitchell Kristy Elizabeth Boyer and James C Lester. Evaluating State Representations for Reinforcement Learning of Turn-Taking Policies in Tutorial Dialogue. In Proceedings of the Fourteenth Annual SIGDIAL Meeting on Discourse and Dialogue (SIGDIAL-2013). 339--343.  Christopher M Mitchell Kristy Elizabeth Boyer and James C Lester. Evaluating State Representations for Reinforcement Learning of Turn-Taking Policies in Tutorial Dialogue. In Proceedings of the Fourteenth Annual SIGDIAL Meeting on Discourse and Dialogue (SIGDIAL-2013). 339--343."},{"key":"e_1_3_2_1_18_1","unstructured":"Philip I Pavlik  Jr Hao Cen and Kenneth R Koedinger. 2009. Performance Factors Analysis--A New Alternative to Knowledge Tracing. Online Submission (2009).  Philip I Pavlik Jr Hao Cen and Kenneth R Koedinger. 2009. Performance Factors Analysis--A New Alternative to Knowledge Tracing. Online Submission (2009)."},{"key":"e_1_3_2_1_19_1","unstructured":"Chris Piech Jonathan Bassen Jonathan Huang Surya Ganguli Mehran Sahami Leonidas J Guibas and Jascha Sohl-Dickstein. 2015. Deep knowledge tracing. In Advances in Neural Information Processing Systems. 505--513.  Chris Piech Jonathan Bassen Jonathan Huang Surya Ganguli Mehran Sahami Leonidas J Guibas and Jascha Sohl-Dickstein. 2015. Deep knowledge tracing. In Advances in Neural Information Processing Systems. 505--513."},{"key":"e_1_3_2_1_20_1","unstructured":"Doina Precup. 2000. Eligibility traces for off-policy policy evaluation. Computer Science Department Faculty Publication Series (2000) 80.  Doina Precup. 2000. Eligibility traces for off-policy policy evaluation. Computer Science Department Faculty Publication Series (2000) 80."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"crossref","unstructured":"Anna N Rafferty Emma Brunskill Thomas L Griffiths and Patrick Shafto. 2015. Faster Teaching via POMDP Planning. Cognitive Science (2015).  Anna N Rafferty Emma Brunskill Thomas L Griffiths and Patrick Shafto. 2015. Faster Teaching via POMDP Planning. Cognitive Science (2015).","DOI":"10.1111\/cogs.12290"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-39112-5_34"},{"key":"e_1_3_2_1_23_1","unstructured":"Joseph Rollinson and Emma Brunskill. 2015. From Predictive Models to Instructional Policies. International Educational Data Mining Society (2015).  Joseph Rollinson and Emma Brunskill. 2015. From Predictive Models to Instructional Policies. International Educational Data Mining Society (2015)."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-19773-9_42"},{"key":"e_1_3_2_1_25_1","unstructured":"Jonathan P Rowe Bradford W Mott and James C Lester. 2014. Optimizing Player Experience in Interactive Narrative Planning: A Modular Reinforcement Learning Approach.. In AIIDE.  Jonathan P Rowe Bradford W Mott and James C Lester. 2014. Optimizing Player Experience in Interactive Narrative Planning: A Modular Reinforcement Learning Approach.. In AIIDE."},{"key":"e_1_3_2_1_26_1","unstructured":"Philip S Thomas and Emma Brunskill. 2016. Data-Efficient Off-Policy Policy Evaluation for Reinforcement Learning. arXiv preprint arXiv:1604.00923 (2016).  Philip S Thomas and Emma Brunskill. 2016. Data-Efficient Off-Policy Policy Evaluation for Reinforcement Learning. arXiv preprint arXiv:1604.00923 (2016)."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"crossref","unstructured":"Philip S Thomas Georgios Theocharous and Mohammad Ghavamzadeh. 2015. High-Confidence Off-Policy Evaluation.. In AAAI. 3000--3006.  Philip S Thomas Georgios Theocharous and Mohammad Ghavamzadeh. 2015. High-Confidence Off-Policy Evaluation.. In AAAI. 3000--3006.","DOI":"10.1609\/aaai.v29i1.9541"},{"key":"e_1_3_2_1_28_1","volume-title":"International Conference on Artificial Intelligence in Education. Springer, 903--905","author":"Yudelson Michael","year":"2015"},{"key":"e_1_3_2_1_29_1","unstructured":"Li Zhou and Emma Brunskill. 2016. Latent Contextual Bandits and their Application to Personalized Recommendations for New Users. arXiv preprint arXiv:1604.06743 (2016).  Li Zhou and Emma Brunskill. 2016. Latent Contextual Bandits and their Application to Personalized Recommendations for New Users. arXiv preprint arXiv:1604.06743 (2016)."}],"event":{"name":"L@S 2017: Fourth (2017) ACM Conference on Learning @ Scale","location":"Cambridge Massachusetts USA","acronym":"L@S 2017","sponsor":["ACM Association for Computing Machinery"]},"container-title":["Proceedings of the Fourth (2017) ACM Conference on Learning @ Scale"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3051457.3051463","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3051457.3051463","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:54:40Z","timestamp":1750222480000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3051457.3051463"}},"subtitle":["Towards a More Principled Offline Exploration of Instructional Policies"],"short-title":[],"issued":{"date-parts":[[2017,4,12]]},"references-count":28,"alternative-id":["10.1145\/3051457.3051463","10.1145\/3051457"],"URL":"https:\/\/doi.org\/10.1145\/3051457.3051463","relation":{},"subject":[],"published":{"date-parts":[[2017,4,12]]},"assertion":[{"value":"2017-04-12","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}