{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,31]],"date-time":"2026-03-31T04:36:54Z","timestamp":1774931814345,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":42,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,3,3]],"date-time":"2025-03-03T00:00:00Z","timestamp":1740960000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"National Science Foundation","award":["2209819"],"award-info":[{"award-number":["2209819"]}]},{"name":"Natural Sciences and Engineering Research Council of Canada","award":["RGPIN-2019-06968"],"award-info":[{"award-number":["RGPIN-2019-06968"]}]},{"name":"Natural Sciences and Engineering Research Council of Canada","award":["RGPIN-2024-04348"],"award-info":[{"award-number":["RGPIN-2024-04348"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,3,3]]},"DOI":"10.1145\/3706468.3706471","type":"proceedings-article","created":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T14:04:11Z","timestamp":1740146651000},"page":"13-23","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Platform-based Adaptive Experimental Research in Education"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2246-0094","authenticated-orcid":false,"given":"Ilya","family":"Musabirov","sequence":"first","affiliation":[{"name":"University of Toronto, Toronto, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9668-3384","authenticated-orcid":false,"given":"Mohi","family":"Reza","sequence":"additional","affiliation":[{"name":"University of Toronto, Toronto, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-7825-4993","authenticated-orcid":false,"given":"Haochen","family":"Song","sequence":"additional","affiliation":[{"name":"University of Toronto, Toronto, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5256-0339","authenticated-orcid":false,"given":"Steven","family":"Moore","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7569-9504","authenticated-orcid":false,"given":"Pan","family":"Chen","sequence":"additional","affiliation":[{"name":"University of Toronto, Toronto, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2878-3986","authenticated-orcid":false,"given":"Harsh","family":"Kumar","sequence":"additional","affiliation":[{"name":"University of Toronto, Toronto, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1345-6675","authenticated-orcid":false,"given":"Tong","family":"Li","sequence":"additional","affiliation":[{"name":"University of Toronto, Toronto, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2291-1468","authenticated-orcid":false,"given":"John","family":"Stamper","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-3666-3973","authenticated-orcid":false,"given":"Norman","family":"Bier","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8319-5370","authenticated-orcid":false,"given":"Anna","family":"Rafferty","sequence":"additional","affiliation":[{"name":"Carleton College, Northfield, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9375-2292","authenticated-orcid":false,"given":"Thomas","family":"Price","sequence":"additional","affiliation":[{"name":"North Carolina State University, Raleigh, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2501-8795","authenticated-orcid":false,"given":"Nina","family":"Deliu","sequence":"additional","affiliation":[{"name":"MEMOTEF, Sapienza University of Rome, Rome, Italy"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9290-8603","authenticated-orcid":false,"given":"Audrey","family":"Durand","sequence":"additional","affiliation":[{"name":"Universit\u00e9 Laval, Quebec City, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2965-5302","authenticated-orcid":false,"given":"Michael","family":"Liut","sequence":"additional","affiliation":[{"name":"University of Toronto Mississauga, Mississauga, Canada and University of Toronto, Toronto, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9122-5242","authenticated-orcid":false,"given":"Joseph Jay","family":"Williams","sequence":"additional","affiliation":[{"name":"University of Toronto, Toronto, Canada"}]}],"member":"320","published-online":{"date-parts":[[2025,3,3]]},"reference":[{"key":"e_1_3_3_2_2_2","doi-asserted-by":"publisher","unstructured":"Micah Allen Davide Poggiali Kirstie Whitaker Tom\u00a0Rhys Marshall Jordy Van\u00a0Langen and Rogier\u00a0A. Kievit. 2021. Raincloud plots: a multi-platform tool for robust data visualization. Wellcome Open Research 4 (Jan. 2021) 63. 10.12688\/wellcomeopenres.15191.2","DOI":"10.12688\/wellcomeopenres.15191.2"},{"key":"e_1_3_3_2_3_2","unstructured":"Susan Athey Undral Byambadalai Vitor Hadad Sanath\u00a0Kumar Krishnamurthy Weiwen Leung and Joseph\u00a0Jay Williams. 2022. Contextual Bandits in a Survey Experiment on Charitable Giving: Within-Experiment Outcomes versus Policy Learning. http:\/\/arxiv.org\/abs\/2211.12004 arXiv:https:\/\/arXiv.org\/abs\/2211.12004 [cs econ stat]."},{"key":"e_1_3_3_2_4_2","volume-title":"Companion Proceedings of the 13th International Learning Analytics and Knowledge Conference","author":"Bier Norman","year":"2023","unstructured":"Norman Bier, John Stamper, Steven Moore, Darren Siegel, and Ariel Anbar. 2023. OLI Torus: a next-generation, open platform for adaptive courseware development, delivery, and research. In Companion Proceedings of the 13th International Learning Analytics and Knowledge Conference. https:\/\/stevenjamesmoore.com\/assets\/papers\/lak23_prac_bier.pdf"},{"key":"e_1_3_3_2_5_2","doi-asserted-by":"publisher","DOI":"10.1109\/CEC48606.2020.9185782"},{"key":"e_1_3_3_2_6_2","doi-asserted-by":"publisher","DOI":"10.5555\/2986459.2986710"},{"key":"e_1_3_3_2_7_2","doi-asserted-by":"publisher","unstructured":"Jacob Cohen. 1992. Statistical Power Analysis. Current Directions in Psychological Science 1 3 (June 1992) 98\u2013101. 10.1111\/1467-8721.ep10768783","DOI":"10.1111\/1467-8721.ep10768783"},{"key":"e_1_3_3_2_8_2","doi-asserted-by":"publisher","unstructured":"Nina Deliu. 2024. Multinomial Thompson sampling for rating scales and prior considerations for calibrating uncertainty. Statistical Methods & Applications 33 2 (April 2024) 439\u2013469. 10.1007\/s10260-023-00732-y","DOI":"10.1007\/s10260-023-00732-y"},{"key":"e_1_3_3_2_9_2","doi-asserted-by":"publisher","unstructured":"Nina Deliu Rajenki Das Angelique May Joseph Newman Jo Steele Melissa Duckworth Rowena\u00a0J. Jones Martin\u00a0R. Wilkins Mark\u00a0R. Toshner and Sofia\u00a0S. Villar. 2024. StratosPHere 2: study protocol for a response-adaptive randomised placebo-controlled phase II trial to evaluate hydroxychloroquine and phenylbutyrate in pulmonary arterial hypertension caused by mutations in BMPR2. Trials 25 1 (Oct. 2024) 680. 10.1186\/s13063-024-08485-z","DOI":"10.1186\/s13063-024-08485-z"},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"publisher","unstructured":"Nina Deliu Joseph\u00a0J. Williams and Sofia\u00a0S. Villar. 2021. Efficient Inference Without Trading-off Regret in Bandits: An Allocation Probability Test for Thompson Sampling. 10.48550\/arXiv.2111.00137arXiv:https:\/\/arXiv.org\/abs\/2111.00137 [stat].","DOI":"10.48550\/arXiv.2111.00137"},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300447"},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.6852930"},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"publisher","DOI":"10.1145\/3027385.3027396"},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"publisher","unstructured":"Kaitlyn\u00a0G. Fitzgerald and Elizabeth Tipton. 2022. The Meta-Analytic Rain Cloud Plot: A New Approach to Visualizing Clearinghouse Data. Journal of Research on Educational Effectiveness 15 4 (Oct. 2022) 848\u2013875. 10.1080\/19345747.2022.2031366","DOI":"10.1080\/19345747.2022.2031366"},{"key":"e_1_3_3_2_15_2","doi-asserted-by":"publisher","unstructured":"Gerd Gigerenzer and Ulrich Hoffrage. 1995. How to improve Bayesian reasoning without instruction: Frequency formats. Psychological Review 102 4 (Oct. 1995) 684\u2013704. 10.1037\/0033-295X.102.4.684","DOI":"10.1037\/0033-295X.102.4.684"},{"key":"e_1_3_3_2_16_2","doi-asserted-by":"publisher","DOI":"10.1145\/3447548.3467124"},{"key":"e_1_3_3_2_17_2","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376454"},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","unstructured":"Rasmus\u00a0Leth J\u00f8rn\u00f8 and Karsten Gynther. 2018. What Constitutes an \u2018Actionable Insight\u2019 in Learning Analytics? Journal of Learning Analytics 5 3 (Dec. 2018). 10.18608\/jla.2018.53.13","DOI":"10.18608\/jla.2018.53.13"},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"crossref","unstructured":"Alex Kale Francis Nguyen Matthew Kay and Jessica Hullman. 2018. Hypothetical outcome plots help untrained observers judge trends in ambiguous data. IEEE transactions on visualization and computer graphics 25 1 (2018) 892\u2013902. Publisher: IEEE.","DOI":"10.1109\/TVCG.2018.2864909"},{"key":"e_1_3_3_2_20_2","doi-asserted-by":"crossref","unstructured":"Michael\u00a0N. Katehakis and Arthur\u00a0F. Veinott. 1987. The Multi-Armed Bandit Problem: Decomposition and Computation. Mathematics of Operations Research 12 2 (1987) 262\u2013268. https:\/\/www.jstor.org\/stable\/3689689","DOI":"10.1287\/moor.12.2.262"},{"key":"e_1_3_3_2_21_2","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858558"},{"key":"e_1_3_3_2_22_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3502053"},{"key":"e_1_3_3_2_23_2","doi-asserted-by":"publisher","unstructured":"Kenneth\u00a0R. Koedinger Julie\u00a0L. Booth and David Klahr. 2013. Instructional Complexity and the Science to Constrain It. Science 342 6161 (Nov. 2013) 935\u2013937. 10.1126\/science.1238056","DOI":"10.1126\/science.1238056"},{"key":"e_1_3_3_2_24_2","doi-asserted-by":"publisher","DOI":"10.1017\/9781108653985"},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","unstructured":"Matthew\u00a0A. Kraft. 2020. Interpreting Effect Sizes of Education Interventions. Educational Researcher 49 4 (May 2020) 241\u2013253. 10.3102\/0013189X20912798Publisher: American Educational Research Association.","DOI":"10.3102\/0013189X20912798"},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"publisher","unstructured":"Matthew\u00a0A. Kraft. 2023. The Effect-Size Benchmark That Matters Most: Education Interventions Often Fail. Educational Researcher 52 3 (April 2023) 183\u2013187. 10.3102\/0013189X231155154Publisher: American Educational Research Association.","DOI":"10.3102\/0013189X231155154"},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i21.30328"},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","unstructured":"Dani\u00ebl Lakens. 2022. Sample Size Justification. Collabra: Psychology 8 1 (March 2022) 33267. 10.1525\/collabra.33267","DOI":"10.1525\/collabra.33267"},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"publisher","unstructured":"Tong Li Jacob Nogas Haochen Song Harsh Kumar Audrey Durand Anna Rafferty Nina Deliu Sofia\u00a0S. Villar and Joseph\u00a0J. Williams. 2022. Algorithms for Adaptive Experiments that Trade-off Statistical Analysis with Reward: Combining Uniform Random Assignment and Reward Maximization. 10.48550\/arXiv.2112.08507arXiv:https:\/\/arXiv.org\/abs\/2112.08507 [cs].","DOI":"10.48550\/arXiv.2112.08507"},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"publisher","DOI":"10.4135\/9781483319605"},{"key":"e_1_3_3_2_31_2","volume-title":"Companion Proceedings 14th International Conference on Learning Analytics & Knowledge (LAK 24)","author":"Musabirov Ilya","year":"2024","unstructured":"Ilya Musabirov, Mohi Reza, Steven Moore, Pan Chen, Harsh Kumar, Tong Li, Haochen Song, Jiakai Shi, Koby Choy, Thomas Price, John Stamper, Norman Bier, Nina Deliu, Sofia Villar, Anna Rafferty, Audrey Durand, and Joseph Williams. 2024. Platform-based Adaptive Experimental Research in Education: Lessons Learned from Digital Learning Challenge. In Companion Proceedings 14th International Conference on Learning Analytics & Knowledge (LAK 24). Kyoto."},{"key":"e_1_3_3_2_32_2","doi-asserted-by":"publisher","DOI":"10.1145\/3660650.3660659"},{"key":"e_1_3_3_2_33_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-51394-2_26"},{"key":"e_1_3_3_2_34_2","unstructured":"Anna Rafferty Huiji Ying and Joseph Williams. 2019. Statistical consequences of using multi-armed bandits to conduct adaptive educational experiments. Journal of Educational Data Mining 11 1 (2019) 47\u201379. https:\/\/jedm.educationaldatamining.org\/index.php\/JEDM\/article\/view\/357"},{"key":"e_1_3_3_2_35_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-93846-2_53"},{"key":"e_1_3_3_2_36_2","doi-asserted-by":"publisher","DOI":"10.1145\/3430895.3460128"},{"key":"e_1_3_3_2_37_2","doi-asserted-by":"publisher","DOI":"10.1145\/3573051.3593397"},{"key":"e_1_3_3_2_38_2","unstructured":"Jaehyeok Shin Aaditya Ramdas and Alessandro Rinaldo. 2019. Are sample means in multi-armed bandits positively or negatively biased? Advances in Neural Information Processing Systems 32 (2019). https:\/\/proceedings.neurips.cc\/paper\/2019\/hash\/65b1e92c585fd4c2159d5f33b5030ff2-Abstract.html"},{"key":"e_1_3_3_2_39_2","unstructured":"Joseph\u00a0Jay Williams Jacob Nogas Nina Deliu Hammad Shaikh Sofia\u00a0S. Villar Audrey Durand and Anna Rafferty. 2021. Challenges in Statistical Analysis of Data Collected by a Bandit Algorithm: An Empirical Exploration in Applications to Adaptively Randomized Experiments. http:\/\/arxiv.org\/abs\/2103.12198"},{"key":"e_1_3_3_2_40_2","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173781"},{"key":"e_1_3_3_2_41_2","doi-asserted-by":"publisher","unstructured":"Alyssa\u00a0F. Wise Simon Knight and Xavier Ochoa. 2021. What Makes Learning Analytics Research Matter. Journal of Learning Analytics 8 3 (Dec. 2021) 1\u20139. 10.18608\/jla.2021.7647","DOI":"10.18608\/jla.2021.7647"},{"key":"e_1_3_3_2_42_2","doi-asserted-by":"publisher","unstructured":"Betsy Wolf and Erica Harbatkin. 2023. Making Sense of Effect Sizes: Systematic Differences in Intervention Effect Sizes by Outcome Measure Type. Journal of Research on Educational Effectiveness 16 1 (Jan. 2023) 134\u2013161. 10.1080\/19345747.2022.2071364","DOI":"10.1080\/19345747.2022.2071364"},{"key":"e_1_3_3_2_43_2","doi-asserted-by":"publisher","unstructured":"Sam Zhang Patrick\u00a0R. Heck Michelle\u00a0N. Meyer Christopher\u00a0F. Chabris Daniel\u00a0G. Goldstein and Jake\u00a0M. Hofman. 2023. An illusion of predictability in scientific results: Even experts confuse inferential uncertainty and outcome variability. Proceedings of the National Academy of Sciences 120 33 (Aug. 2023) e2302491120. 10.1073\/pnas.2302491120","DOI":"10.1073\/pnas.2302491120"}],"event":{"name":"LAK '25: The 15th International Learning Analytics and Knowledge Conference","location":"Dublin Ireland","acronym":"LAK 2025"},"container-title":["Proceedings of the 15th International Learning Analytics and Knowledge Conference"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706468.3706471","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3706468.3706471","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:18:04Z","timestamp":1750295884000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706468.3706471"}},"subtitle":["Lessons Learned from The Digital Learning Challenge"],"short-title":[],"issued":{"date-parts":[[2025,3,3]]},"references-count":42,"alternative-id":["10.1145\/3706468.3706471","10.1145\/3706468"],"URL":"https:\/\/doi.org\/10.1145\/3706468.3706471","relation":{},"subject":[],"published":{"date-parts":[[2025,3,3]]},"assertion":[{"value":"2025-03-03","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}