{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,23]],"date-time":"2026-06-23T03:14:47Z","timestamp":1782184487802,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":31,"publisher":"ACM","funder":[{"DOI":"10.13039\/501100006374","name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","doi-asserted-by":"publisher","award":["VI.Veni.211G.016"],"award-info":[{"award-number":["VI.Veni.211G.016"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,7,17]]},"DOI":"10.1145\/3698205.3729553","type":"proceedings-article","created":{"date-parts":[[2025,7,17]],"date-time":"2025-07-17T14:35:08Z","timestamp":1752762908000},"page":"13-23","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Assessing Reliability in AI-Powered Learning Systems with A\/A Tests"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-6394-8065","authenticated-orcid":false,"given":"Lilian","family":"Ye","sequence":"first","affiliation":[{"name":"Prowise Learn, Amsterdam, Netherlands and Psychological Methods, University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9271-7476","authenticated-orcid":false,"given":"Alexander O.","family":"Savi","sequence":"additional","affiliation":[{"name":"Psychological Methods, University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4269-5296","authenticated-orcid":false,"given":"Abe D.","family":"Hofman","sequence":"additional","affiliation":[{"name":"Psychological Methods, University of Amsterdam, Amsterdam, Netherlands and Prowise Learn, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,7,17]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.18608\/jla.2018.52.3"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1557019.1557139"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1547-5069.2007.00161.x"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0121945"},{"key":"e_1_3_2_2_5_1","volume-title":"The rating of chess players past and present","author":"Elo A","year":"1978","unstructured":"A Elo. 1978. The rating of chess players past and present. New York: Arco Publishing (1978)."},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.3390\/educsci13121216"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2015.7363863"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.5964\/meth.9263"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1080\/08982112.2022.2089854"},{"key":"e_1_3_2_2_10_1","volume-title":"Causal inference in statistics, social, and biomedical sciences","author":"Imbens Guido W","unstructured":"Guido W Imbens and Donald B Rubin. 2015. Causal inference in statistics, social, and biomedical sciences. Cambridge university press."},{"key":"e_1_3_2_2_11_1","unstructured":"JASP Team. 2024. JASP (Version 0.19.0)[Computer software]. https:\/\/jasp-stats.org\/"},{"key":"e_1_3_2_2_12_1","volume-title":"Always valid inference: Bringing sequential analysis to A\/B testing. arXiv preprint arXiv:1512.04922","author":"Johari Ramesh","year":"2015","unstructured":"Ramesh Johari, Leo Pekelis, and David J Walsh. 2015. Always valid inference: Bringing sequential analysis to A\/B testing. arXiv preprint arXiv:1512.04922 (2015)."},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3636555.3636858"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.compedu.2011.02.003"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1281192.1281295"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/1964897.1964905"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4899-7687-1_891"},{"key":"e_1_3_2_2_18_1","volume-title":"Controlled experiments on the web: survey and practical guide. Data mining and knowledge discovery","author":"Kohavi Ron","year":"2009","unstructured":"Ron Kohavi, Roger Longbotham, Dan Sommerfield, and Randal M Henne. 2009. Controlled experiments on the web: survey and practical guide. Data mining and knowledge discovery, Vol. 18 (2009), 140-181."},{"key":"e_1_3_2_2_19_1","volume-title":"Statistical theories of mental test scores","author":"Lord Frederic M","unstructured":"Frederic M Lord and Melvin R Novick. 1968. Statistical theories of mental test scores. Addison-Wesley."},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.3390\/fi12080124"},{"key":"e_1_3_2_2_21_1","volume-title":"Proceedings of the 7th International Conference on Educational Data Mining. 6-13","author":"Papou\u0161ek Jan","year":"2014","unstructured":"Jan Papou\u0161ek, Radek Pel\u00e1nek, and V\u00edt Stanislav. 2014. Adaptive practice of facts in domains with varied prior knowledge. In Proceedings of the 7th International Conference on Educational Data Mining. 6-13."},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.compedu.2016.03.017"},{"key":"e_1_3_2_2_23_1","volume-title":"R: A Language and Environment for Statistical Computing","author":"Team R Core","year":"2024","unstructured":"R Core Team. 2024. R: A Language and Environment for Statistical Computing. R Foundation for Statistical Computing, Vienna, Austria. https:\/\/www.R-project.org\/"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2883851.2883876"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.compedu.2017.12.008"},{"key":"e_1_3_2_2_26_1","volume-title":"Nick ten Broeke, and Abe D Hofman","author":"Savi Alexander O","year":"2021","unstructured":"Alexander O Savi, Nick ten Broeke, and Abe D Hofman. 2021. Adaptive learning systems and interference in causal inference. In Educational A\/B Testing at Scale (2021-06-22). https:\/\/sites.google.com\/carnegielearning.com\/edu-ab-testing-at-scale-2021\/"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.31219\/osf.io\/83jsg"},{"key":"e_1_3_2_2_28_1","first-page":"701","volume-title":"Science","volume":"171","author":"Shepard Roger N","year":"1971","unstructured":"Roger N Shepard and Jacqueline Metzler. 1971. Mental rotation of three-dimensional objects. Science, Vol. 171, 3972 (1971), 701-703."},{"key":"e_1_3_2_2_29_1","volume-title":"The most powerful way to turn clicks into customers","author":"Siroker Dan","unstructured":"Dan Siroker and Pete Koomen. 2015. A\/B testing: The most powerful way to turn clicks into customers. John Wiley and Sons."},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.31234\/osf.io\/6hzke"},{"key":"e_1_3_2_2_31_1","volume-title":"Han LJ van der Maas, and Abe D Hofman","author":"Vermeiren Hanke","year":"2025","unstructured":"Hanke Vermeiren, Joost Kruis, Maria Bolsinova, Han LJ van der Maas, and Abe D Hofman. 2025. Psychometrics of an Elo-based Large-Scale Online Learning System. Computers and Education: Artificial Intelligence (2025), 100376."}],"event":{"name":"L@S '25: Twelfth ACM Conference on Learning @ Scale","location":"Palermo Italy","acronym":"L@S '25"},"container-title":["Proceedings of the Twelfth ACM Conference on Learning @ Scale"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3698205.3729553","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T13:16:12Z","timestamp":1755868572000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3698205.3729553"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,17]]},"references-count":31,"alternative-id":["10.1145\/3698205.3729553","10.1145\/3698205"],"URL":"https:\/\/doi.org\/10.1145\/3698205.3729553","relation":{},"subject":[],"published":{"date-parts":[[2025,7,17]]},"assertion":[{"value":"2025-07-17","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}