{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,31]],"date-time":"2026-03-31T04:36:56Z","timestamp":1774931816661,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":45,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,5,2]],"date-time":"2024-05-02T00:00:00Z","timestamp":1714608000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100006374","name":"Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"publisher","award":["RGPIN-2019-06968"],"award-info":[{"award-number":["RGPIN-2019-06968"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100006374","name":"Office of Naval Research","doi-asserted-by":"publisher","award":["N00014-21-1-2576"],"award-info":[{"award-number":["N00014-21-1-2576"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100006374","name":"National Science Foundation","doi-asserted-by":"publisher","award":["2209819"],"award-info":[{"award-number":["2209819"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,5,2]]},"DOI":"10.1145\/3660650.3660659","type":"proceedings-article","created":{"date-parts":[[2024,6,20]],"date-time":"2024-06-20T06:20:39Z","timestamp":1718864439000},"page":"1-7","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Opportunities for Adaptive Experiments to Enable Continuous Improvement in Computer Science Education"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2246-0094","authenticated-orcid":false,"given":"Ilya","family":"Musabirov","sequence":"first","affiliation":[{"name":"Department of Computer Science, University of Toronto, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1228-5774","authenticated-orcid":false,"given":"Angela","family":"Zavaleta Bernuy","sequence":"additional","affiliation":[{"name":"Department of Computer Science, University of Toronto, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7569-9504","authenticated-orcid":false,"given":"Pan","family":"Chen","sequence":"additional","affiliation":[{"name":"Department of Computer Science, University of Toronto, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2965-5302","authenticated-orcid":false,"given":"Michael","family":"Liut","sequence":"additional","affiliation":[{"name":"Mathematical and Computational Sciences, University of Toronto Mississauga, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9122-5242","authenticated-orcid":false,"given":"Joseph","family":"Williams","sequence":"additional","affiliation":[{"name":"Department of Computer Science, University of Toronto, Canada"}]}],"member":"320","published-online":{"date-parts":[[2024,6,20]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1606.03966"},{"key":"e_1_3_2_1_2_1","volume-title":"Analysis of Thompson Sampling for the Multi-armed Bandit Problem. In Conference on learning theory. JMLR Workshop and Conference Proceedings, 39\u20131.","author":"Agrawal Shipra","year":"2012","unstructured":"Shipra Agrawal and Navin Goyal. 2012. Analysis of Thompson Sampling for the Multi-armed Bandit Problem. In Conference on learning theory. JMLR Workshop and Conference Proceedings, 39\u20131."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/2839509.2844601"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1007\/s40593-015-0089-1"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3430895.3460145"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3219166.3219204"},{"key":"e_1_3_2_1_7_1","volume-title":"Conference on Neural Information Processing Systems. 1\u20138.","author":"Bakshy Eytan","year":"2018","unstructured":"Eytan Bakshy, Lili Dworkin, Brian Karrer, Konstantin Kashin, Benjamin Letham, Ashwin Murthy, and Shaun Singh. 2018. AE: A domain-agnostic platform for adaptive experimentation. In Conference on Neural Information Processing Systems. 1\u20138."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2676723.2677282"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0234817"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-94-007-1576-9_7"},{"key":"e_1_3_2_1_11_1","volume-title":"Advances in Neural Information Processing Systems, J.\u00a0Shawe-Taylor, R.\u00a0Zemel, P.\u00a0Bartlett, F.\u00a0Pereira, and K.\u00a0Q","author":"Chapelle Olivier","year":"2011","unstructured":"Olivier Chapelle and Lihong Li. 2011. An Empirical Evaluation of Thompson Sampling. In Advances in Neural Information Processing Systems, J.\u00a0Shawe-Taylor, R.\u00a0Zemel, P.\u00a0Bartlett, F.\u00a0Pereira, and K.\u00a0Q. Weinberger (Eds.). Vol.\u00a024. Curran Associates, Inc.https:\/\/proceedings.neurips.cc\/paper\/2011\/file\/e53a0a2978c28872a4505bdb51db06dc-Paper.pdf"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1080\/0899340042000303474"},{"key":"e_1_3_2_1_13_1","unstructured":"Nina Deliu Joseph\u00a0J. Williams and Sofia\u00a0S. Villar. 2021. Efficient Inference Without Trading-off Regret in Bandits: An Allocation Probability Test for Thompson Sampling. http:\/\/arxiv.org\/abs\/2111.00137 arXiv:2111.00137 [cs stat]."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3082060"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1080\/0144929X.2019.1669712"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.6852930"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2014602118"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","unstructured":"Martin\u00a0S. Hagger Linda\u00a0D. Cameron Kyra Hamilton Nelli Hankonen and Taru Lintunen (Eds.). 2020. The Handbook of Behavior Change (1 ed.). Cambridge University Press. https:\/\/doi.org\/10.1017\/9781108677318","DOI":"10.1017\/9781108677318"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3470652"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1007\/s40593-014-0024-x"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3375462.3375489"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287324.3287420"},{"key":"e_1_3_2_1_23_1","volume-title":"Trustworthy online controlled experiments: a practical guide to A\/B testing","author":"Kohavi Ron","unstructured":"Ron Kohavi, Diane Tang, and Ya Xu. 2020. Trustworthy online controlled experiments: a practical guide to A\/B testing. Cambridge University Press, Cambridge, United Kingdom ; New York, NY."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1186\/s13063-020-4084-y"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3441636.3442302"},{"key":"e_1_3_2_1_26_1","unstructured":"Tong Li Jacob Nogas Haochen Song Harsh Kumar Audrey Durand Anna Rafferty Nina Deliu Sofia\u00a0S. Villar and Joseph\u00a0J. Williams. 2022. Algorithms for Adaptive Experiments that Trade-off Statistical Analysis with Reward: Combining Uniform Random Assignment and Reward Maximization. arxiv:2112.08507\u00a0[cs.LG]"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3478431.3499358"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2960310.2960328"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3478431.3499418"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3372782.3406264"},{"key":"e_1_3_2_1_31_1","first-page":"236","article-title":"Learning analytics as a tool for closing the assessment loop in higher education. Knowledge management & e-learning","volume":"4","author":"Mattingly D.","year":"2012","unstructured":"Karen\u00a0D. Mattingly, Margaret\u00a0C. Rice, and Zane\u00a0L. Berge. 2012. Learning analytics as a tool for closing the assessment loop in higher education. Knowledge management & e-learning: An international journal 4, 3 (2012), 236\u2013247.","journal-title":"An international journal"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.4324\/9781315105642"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.18608\/jla.2018.52.4"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2737596"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1080\/10824669.2014.988334"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.5281\/ZENODO.3554749"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3430895.3460128"},{"key":"e_1_3_2_1_38_1","volume-title":"Third Annual Workshop on A\/B Testing and Platform-Enabled Learning Research. In Proceedings of the Ninth ACM Conference on Learning@ Scale. 252\u2013254","author":"Ritter Steven","year":"2022","unstructured":"Steven Ritter, Neil Heffernan, Joseph\u00a0Jay Williams, Derek Lomas, Ben Motz, Debshila Basu\u00a0Mallick, Klinton Bicknell, Danielle McNamara, Rene\u00a0F. Kizilcec, and Jeremy Roschelle. 2022. Third Annual Workshop on A\/B Testing and Platform-Enabled Learning Research. In Proceedings of the Ninth ACM Conference on Learning@ Scale. 252\u2013254."},{"key":"e_1_3_2_1_39_1","volume-title":"Proceedings of the 5th International Conference on Educational Data Mining. International Educational Data Mining Society","author":"Stamper J.C.","year":"2012","unstructured":"J.C. Stamper, D. Lomas, D. Ching, S. Ritter, Koedinger, and J. K.R., Steinhart. 2012. The rise of the super experiment. In Proceedings of the 5th International Conference on Educational Data Mining. International Educational Data Mining Society, Chania, Greece, 196\u2013199."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.cedpsych.2020.101872"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/2876034.2876042"},{"key":"e_1_3_2_1_42_1","unstructured":"Joseph\u00a0Jay Williams Jacob Nogas Nina Deliu Hammad Shaikh Sofia\u00a0S. Villar Audrey Durand and Anna Rafferty. 2021. Challenges in Statistical Analysis of Data Collected by a Bandit Algorithm: An Empirical Exploration in Applications to Adaptively Randomized Experiments. http:\/\/arxiv.org\/abs\/2103.12198"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173781"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","unstructured":"Fernando\u00a0J. Yanez Angela Zavaleta-Bernuy Ziwen Han Michael Liut Anna Rafferty and Joseph\u00a0Jay Williams. 2022. Increasing Students\u2019 Engagement to Reminder Emails Through Multi-Armed Bandits. https:\/\/doi.org\/10.48550\/arXiv.2208.05090","DOI":"10.48550\/arXiv.2208.05090"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-78270-2_75"}],"event":{"name":"WCCCE '24: The 26th Western Canadian Conference on Computing Education","location":"Kelowna BC Canada","acronym":"WCCCE '24"},"container-title":["The 26th Western Canadian Conference on Computing Education"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3660650.3660659","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3660650.3660659","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,26]],"date-time":"2025-08-26T19:49:36Z","timestamp":1756237776000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3660650.3660659"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5,2]]},"references-count":45,"alternative-id":["10.1145\/3660650.3660659","10.1145\/3660650"],"URL":"https:\/\/doi.org\/10.1145\/3660650.3660659","relation":{},"subject":[],"published":{"date-parts":[[2024,5,2]]},"assertion":[{"value":"2024-06-20","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}