{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,22]],"date-time":"2026-03-22T08:00:45Z","timestamp":1774166445775,"version":"3.50.1"},"reference-count":101,"publisher":"Elsevier BV","issue":"1","license":[{"start":{"date-parts":[[2021,5,4]],"date-time":"2021-05-04T00:00:00Z","timestamp":1620086400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2021,5,4]],"date-time":"2021-05-04T00:00:00Z","timestamp":1620086400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Int J Artif Intell Educ"],"published-print":{"date-parts":[[2022,3]]},"DOI":"10.1007\/s40593-021-00252-4","type":"journal-article","created":{"date-parts":[[2021,5,4]],"date-time":"2021-05-04T17:02:50Z","timestamp":1620147770000},"page":"196-232","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":18,"title":["Complexity and Difficulty of Items in Learning Systems"],"prefix":"10.1016","volume":"32","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8877-4729","authenticated-orcid":false,"given":"Radek","family":"Pel\u00e1nek","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5601-926X","authenticated-orcid":false,"given":"Tom\u00e1\u0161","family":"Effenberger","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8673-8705","authenticated-orcid":false,"given":"Jaroslav","family":"\u010cech\u00e1k","sequence":"additional","affiliation":[]}],"member":"78","published-online":{"date-parts":[[2021,5,4]]},"reference":[{"key":"252_CR1","unstructured":"Aleven, V., McLaughlin, E. A., Glenn, R. A., & Koedinger, K. R. (2016). Instruction based on adaptive learning technologies. In Handbook of research on learning and instruction. Routledge."},{"issue":"2","key":"252_CR2","first-page":"157","volume":"26","author":"A Alvarez","year":"2010","unstructured":"Alvarez, A., & Scott, T. A. (2010). Using student surveys in determining the difficulty of programming assignments. Journal of Computing Sciences in Colleges, 26(2), 157\u2013163.","journal-title":"Journal of Computing Sciences in Colleges"},{"issue":"1","key":"252_CR3","doi-asserted-by":"publisher","first-page":"121","DOI":"10.1007\/s10648-017-9398-2","volume":"30","author":"SJ Amendum","year":"2018","unstructured":"Amendum, S. J., Conradi, K., & Hiebert, E. (2018). Does text complexity matter in the elementary grades? A research synthesis of text difficulty and elementary students\u2019 reading fluency and comprehension. Educational Psychology Review, 30(1), 121\u2013151.","journal-title":"Educational Psychology Review"},{"key":"252_CR4","unstructured":"Anderson, L. W., Krathwohl, D. R., Airasian, P. W., Cruikshank, K. A., Mayer, R. E., Pintrich, P. R., Raths J., & Wittrock, M. C. (2000). A taxonomy for learning, teaching, and assessing: A revision of Bloom\u2019s taxonomy of educational objectives., abridged edition. Pearson."},{"issue":"4","key":"252_CR5","doi-asserted-by":"publisher","first-page":"205","DOI":"10.1016\/j.entcom.2011.04.001","volume":"2","author":"M-V Aponte","year":"2011","unstructured":"Aponte, M.-V., Levieux, G., & Natkin, S. (2011). Measuring the level of difficulty in single player video games. Entertainment Computing, 2(4), 205\u2013213.","journal-title":"Entertainment Computing"},{"issue":"2","key":"252_CR6","doi-asserted-by":"publisher","first-page":"153","DOI":"10.1080\/08957340701301272","volume":"20","author":"ME Ascalon","year":"2007","unstructured":"Ascalon, M. E., Meyers, L. S., Davis, B. W., & Smits, N. (2007). Distractor similarity and item-stem structure: Effects on item difficulty. Applied Measurement in Education, 20(2), 153\u2013170.","journal-title":"Applied Measurement in Education"},{"issue":"2","key":"252_CR7","first-page":"279","volume":"47","author":"HN Ayako Hoshino","year":"2010","unstructured":"Ayako Hoshino, H. N. (2010). Predicting the difficulty of multiple-choice close questions for computer-adaptive testing. Research in Computing Science, 47(2), 279\u2013292.","journal-title":"Research in Computing Science"},{"issue":"3","key":"252_CR8","doi-asserted-by":"publisher","first-page":"285","DOI":"10.1016\/S0271-5309(01)00005-2","volume":"21","author":"A Bailin","year":"2001","unstructured":"Bailin, A., & Grafstein, A. (2001). The linguistic assumptions underlying readability formulae: A critique. Language & Communication, 21(3), 285\u2013301.","journal-title":"Language & Communication"},{"key":"252_CR9","unstructured":"Baker, F. B. (2001). The basics of item response theory. ERIC."},{"issue":"2","key":"252_CR10","doi-asserted-by":"publisher","first-page":"600","DOI":"10.1007\/s40593-016-0105-0","volume":"26","author":"RS Baker","year":"2016","unstructured":"Baker, R. S. (2016). Stupid tutoring systems, intelligent humans. International Journal of Artificial Intelligence in Education, 26(2), 600\u2013614.","journal-title":"International Journal of Artificial Intelligence in Education"},{"issue":"2","key":"252_CR11","first-page":"185","volume":"19","author":"R Baker","year":"2008","unstructured":"Baker, R., Walonoski, J., Heffernan, N., Roll, I., Corbett, A., & Koedinger, K. (2008). Why students engage in gaming the system behavior in interactive learning environments. Journal of Interactive Learning Research, 19(2), 185\u2013224.","journal-title":"Journal of Interactive Learning Research"},{"key":"252_CR12","doi-asserted-by":"crossref","unstructured":"Baldwin, P., Yaneva, V., Mee, J., Clauser, B. E., & Ha, L. A. (2020). Using natural language processing to predict item response times and improve test construction. Journal of Educational Measurement.","DOI":"10.1111\/jedm.12264"},{"issue":"2","key":"252_CR13","doi-asserted-by":"publisher","first-page":"1","DOI":"10.5296\/ije.v2i2.508","volume":"2","author":"OC Barbu","year":"2010","unstructured":"Barbu, O. C., & Beal, C. R. (2010). Effects of linguistic complexity and math difficulty on word problem solving by english learners. International Journal of Education, 2(2), 1\u201319.","journal-title":"International Journal of Education"},{"issue":"6","key":"252_CR14","doi-asserted-by":"publisher","first-page":"72","DOI":"10.1145\/3015455","volume":"60","author":"D Bau","year":"2017","unstructured":"Bau, D., Gray, J., Kelleher, C., Sheldon, J., & Turbak, F. (2017). Learnable programming: Blocks and beyond. Communications of the ACM, 60(6), 72\u201380.","journal-title":"Communications of the ACM"},{"issue":"2","key":"252_CR15","doi-asserted-by":"publisher","first-page":"15","DOI":"10.3390\/jintelligence5020015","volume":"5","author":"JF Beckmann","year":"2017","unstructured":"Beckmann, J. F., & Goode, N. (2017). Missing the wood for the wrong trees: On the difficulty of defining the complexity of complex problem solving scenarios. Journal of Intelligence, 5(2), 15.","journal-title":"Journal of Intelligence"},{"key":"252_CR16","doi-asserted-by":"publisher","first-page":"1739","DOI":"10.3389\/fpsyg.2017.01739","volume":"8","author":"JF Beckmann","year":"2017","unstructured":"Beckmann, J. F., Birney, D. P., & Goode, N. (2017). Beyond psychometrics: The difference between difficult problem solving and complex problem solving. Frontiers in Psychology, 8, 1739.","journal-title":"Frontiers in Psychology"},{"key":"252_CR17","doi-asserted-by":"crossref","unstructured":"Benedetto, L., Cappelli, A., Turrin, R., & Cremonesi, P. (2020). R2de: A NLP approach to estimating IRT parameters of newly generated questions. In Proceedings of learning analytics & knowledge.","DOI":"10.1145\/3375462.3375517"},{"issue":"1","key":"252_CR18","doi-asserted-by":"publisher","first-page":"63","DOI":"10.1007\/s10648-011-9181-8","volume":"24","author":"RG Benjamin","year":"2012","unstructured":"Benjamin, R. G. (2012). Reconstructing readability: Recent developments and recommendations in the analysis of text difficulty. Educational Psychology Review, 24(1), 63\u201388.","journal-title":"Educational Psychology Review"},{"key":"252_CR19","unstructured":"Biggs, J. B., & Collis, K. F. (1981). Evaluating the quality of learning: The SOLO taxonomy (structure of the observed learning outcome). Academic Press."},{"key":"252_CR20","unstructured":"Bloom, B. S., Engelhart, M. B., Furst, E. J., Hill, W. H., & Krathwohl, D. R. (1956). Taxonomy of educational objectives. In The classification of educational goals. Handbook 1: Cognitive domain. Longmans Green."},{"key":"252_CR21","doi-asserted-by":"crossref","unstructured":"Bouvier, D., Lovellette, E., Matta, J., Alshaigy, B., Becker, B. A., Craig, M., Jackova, J., McCartney, R., Sanders, K., & Zarb, M. Novice programmers and the problem description effect. In Proceedings of the 2016 ITiCSE working group reports, ITiCSE \u201816 (pp. 103\u2013118). ACM.","DOI":"10.1145\/3024906.3024912"},{"key":"252_CR22","doi-asserted-by":"crossref","unstructured":"Brooks, C., Chavez, O., Tritz, J., & Teasley, S. (2015). Reducing selection bias in quasi-experimental educational studies. In Proceedings of learning analytics & knowledge (pp. 295\u2013299). ACM.","DOI":"10.1145\/2723576.2723614"},{"key":"252_CR23","doi-asserted-by":"crossref","unstructured":"Brusilovsky, P. L. (1992). A framework for intelligent knowledge sequencing and task sequencing. In Proceedings of intelligent tutoring systems (pp. 499\u2013506). Springer.","DOI":"10.1007\/3-540-55606-0_59"},{"issue":"1","key":"252_CR24","doi-asserted-by":"publisher","first-page":"40","DOI":"10.2307\/258353","volume":"13","author":"DJ Campbell","year":"1988","unstructured":"Campbell, D. J. (1988). Task complexity: A review and analysis. Academy of Management Review, 13(1), 40\u201352.","journal-title":"Academy of Management Review"},{"key":"252_CR25","doi-asserted-by":"crossref","unstructured":"\u010cech\u00e1k, J., & Pel\u00e1nek, R. (2019). Item ordering biases in educational data. In S. Isotani, E. Mill\u00e1n, A. Ogan, P. Hastings, B. McLaren, & R. Luckin (Eds.), Proceedings of artificial intelligence in education (pp. 48\u201358). Springer.","DOI":"10.1007\/978-3-030-23204-7_5"},{"key":"252_CR26","doi-asserted-by":"crossref","unstructured":"Cen, H., Koedinger, K., & Junker, B. (2006). Learning factors analysis\u2013a general method for cognitive model evaluation and improvement. In Proceedings of intelligent tutoring systems (pp. 164\u2013175). Springer.","DOI":"10.1007\/11774303_17"},{"issue":"2","key":"252_CR27","doi-asserted-by":"publisher","first-page":"378","DOI":"10.1016\/j.eswa.2005.07.029","volume":"30","author":"C-M Chen","year":"2006","unstructured":"Chen, C.-M., Liu, C.-Y., & Chang, M.-H. (2006). Personalized curriculum sequencing utilizing modified item response theory for web-based instruction. Expert Systems with Applications, 30(2), 378\u2013396.","journal-title":"Expert Systems with Applications"},{"key":"252_CR28","doi-asserted-by":"crossref","unstructured":"Craig, M., Smith, J., & Petersen, A. (2017). Familiar contexts and the difficulty of programming problems. In Proceedings of computing education research (pp. 123\u2013127). ACM.","DOI":"10.1145\/3141880.3141898"},{"key":"252_CR29","unstructured":"Csikszentmihalyi, M., & Csikszentmihalyi, I. S. (1992). Optimal experience: Psychological studies of flow in consciousness. Cambridge University Press."},{"key":"252_CR30","doi-asserted-by":"publisher","first-page":"348","DOI":"10.3389\/fpsyg.2015.00348","volume":"6","author":"G Daroczy","year":"2015","unstructured":"Daroczy, G., Wolska, M., Meurers, W. D., & Nuerk, H.-C. (2015). Word problems: A review of linguistic and numerical factors contributing to their difficulty. Frontiers in Psychology, 6, 348.","journal-title":"Frontiers in Psychology"},{"key":"252_CR31","unstructured":"De Ayala, R. (2008). The theory and practice of item response theory. The Guilford Press."},{"key":"252_CR32","doi-asserted-by":"crossref","unstructured":"Eagle, M., & Barnes, T. (2014). Survival analysis on duration data in intelligent tutors. In Proceedings of intelligent tutoring systems (pp. 178\u2013187). Springer.","DOI":"10.1007\/978-3-319-07221-0_22"},{"key":"252_CR33","doi-asserted-by":"crossref","unstructured":"Effenberger, T., \u010cech\u00e1k, J., & Pel\u00e1nek, R. (2019). Measuring difficulty of introductory programming tasks. In Proceedings learning at scale, pp. 1\u20134.","DOI":"10.1145\/3330430.3333641"},{"issue":"3","key":"252_CR34","doi-asserted-by":"publisher","first-page":"225","DOI":"10.1111\/j.1745-3984.2008.00062.x","volume":"45","author":"H Finch","year":"2008","unstructured":"Finch, H. (2008). Estimation of item response theory parameters in the presence of missing data. Journal of Educational Measurement, 45(3), 225\u2013245.","journal-title":"Journal of Educational Measurement"},{"issue":"6","key":"252_CR35","doi-asserted-by":"publisher","first-page":"1082","DOI":"10.3102\/0034654317726529","volume":"87","author":"MJ Gierl","year":"2017","unstructured":"Gierl, M. J., Bulut, O., Guo, Q., & Zhang, X. (2017). Developing, analyzing, and using distractors for multiple-choice tests in education: A comprehensive review. Review of Educational Research, 87(6), 1082\u20131116.","journal-title":"Review of Educational Research"},{"key":"252_CR36","doi-asserted-by":"crossref","unstructured":"Gluga, R., Kay, J., Lister, R., Kleitman, S., & Lever, T. (2012). Coming to terms with Bloom: An online tutorial for teachers of programming fundamentals. In Proceedings of Australasian computing education conference (pp. 147\u2013156). Australian Computer Society, Inc.","DOI":"10.1145\/2157136.2157181"},{"key":"252_CR37","doi-asserted-by":"crossref","unstructured":"Goutte, C., Durand, G., & L\u00e9ger, S. (2018). On the learning curve attrition bias in additive factor modeling. In Proceedings of artificial intelligence in education (pp. 109\u2013113). Springer.","DOI":"10.1007\/978-3-319-93846-2_21"},{"issue":"2","key":"252_CR38","doi-asserted-by":"publisher","first-page":"193","DOI":"10.3758\/BF03195564","volume":"36","author":"AC Graesser","year":"2004","unstructured":"Graesser, A. C., McNamara, D. S., Louwerse, M. M., & Cai, Z. (2004). Coh-metrix: Analysis of text on cohesion and language. Behavior Research Methods, Instruments, & Computers, 36(2), 193\u2013202.","journal-title":"Behavior Research Methods, Instruments, & Computers"},{"issue":"3\u20134","key":"252_CR39","doi-asserted-by":"publisher","first-page":"321","DOI":"10.1093\/biomet\/28.3-4.321","volume":"28","author":"H Hotelling","year":"1936","unstructured":"Hotelling, H. (1936). Relations between two sets of variates. Biometrika, 28(3\u20134), 321\u2013377.","journal-title":"Biometrika"},{"key":"252_CR40","doi-asserted-by":"crossref","unstructured":"Huang, Y., Aleven, V., McLaughlin, E., & Koedinger, K. (2020). A general multi-method approach to design-loop adaptivity in intelligent tutoring systems. In Proceedings of artificial intelligence in education (pp. 124\u2013129). Springer.","DOI":"10.1007\/978-3-030-52240-7_23"},{"key":"252_CR41","doi-asserted-by":"crossref","unstructured":"Hufkens, L. V., & Browne, C. (2019). A functional taxonomy of logic puzzles. In IEEE conference on games (CoG) (Vol. 2019, pp. 1\u20134). IEEE.","DOI":"10.1109\/CIG.2019.8848107"},{"key":"252_CR42","doi-asserted-by":"crossref","unstructured":"Ihantola, P., & Petersen, A. (2019). Code complexity in introductory programming courses. In Proceedings of international conference on system sciences.","DOI":"10.24251\/HICSS.2019.924"},{"key":"252_CR43","unstructured":"Jaru\u0161ek, P., & Pel\u00e1nek, R. (2011). What determines difficulty of transport puzzles? In Proceedings of Florida artificial intelligence research society conference (pp. 428\u2013433). AAAI Press."},{"key":"252_CR44","doi-asserted-by":"crossref","unstructured":"Jumaat, N. F., & Tasir, Z. (2014). Instructional scaffolding in online learning environment: A meta-analysis. In Proceedings of teaching and learning in computing and engineering (pp. 74\u201377). IEEE.","DOI":"10.1109\/LaTiCE.2014.22"},{"key":"252_CR45","doi-asserted-by":"crossref","unstructured":"Kelleher, C., & Hnin, W. (2019). Predicting cognitive load in future code puzzles. In Proceedings of conference on human factors in computing systems (pp. 1\u201312).","DOI":"10.1145\/3290605.3300487"},{"issue":"3","key":"252_CR46","doi-asserted-by":"publisher","first-page":"207","DOI":"10.1207\/S15324818AME1603_3","volume":"16","author":"LA Keller","year":"2003","unstructured":"Keller, L. A., Swaminathan, H., & Sireci, S. G. (2003). Evaluating scoring procedures for context-dependent item sets. Applied Measurement in Education, 16(3), 207\u2013222.","journal-title":"Applied Measurement in Education"},{"key":"252_CR47","doi-asserted-by":"crossref","unstructured":"Khodeir, N. A., Elazhary, H., & Wanas, N. (2018). Generating story problems via controlled parameters in a web-based intelligent tutoring system. The International Journal of Information and Learning Technology.","DOI":"10.1108\/IJILT-09-2017-0085"},{"key":"252_CR48","doi-asserted-by":"publisher","first-page":"78","DOI":"10.1016\/j.procs.2012.10.060","volume":"15","author":"K Kiili","year":"2012","unstructured":"Kiili, K., De Freitas, S., Arnab, S., & Lainema, T. (2012). The design principles for flow experience in educational games. Procedia Computer Science, 15, 78\u201391.","journal-title":"Procedia Computer Science"},{"issue":"2","key":"252_CR49","doi-asserted-by":"publisher","first-page":"129","DOI":"10.1207\/s15327809jls1302_1","volume":"13","author":"KR Koedinger","year":"2004","unstructured":"Koedinger, K. R., & Nathan, M. J. (2004). The real story behind story problems: Effects of representations on quantitative reasoning. The Journal of the Learning Sciences, 13(2), 129\u2013164.","journal-title":"The Journal of the Learning Sciences"},{"issue":"5","key":"252_CR50","doi-asserted-by":"publisher","first-page":"757","DOI":"10.1111\/j.1551-6709.2012.01245.x","volume":"36","author":"KR Koedinger","year":"2012","unstructured":"Koedinger, K. R., Corbett, A. T., & Perfetti, C. (2012). The knowledge-learning-instruction framework: Bridging the science-practice chasm to enhance robust student learning. Cognitive Science, 36(5), 757\u2013798.","journal-title":"Cognitive Science"},{"issue":"2","key":"252_CR51","doi-asserted-by":"publisher","first-page":"248","DOI":"10.1016\/0010-0285(85)90009-X","volume":"17","author":"K Kotovsky","year":"1985","unstructured":"Kotovsky, K., Hayes, J. R., & Simon, H. A. (1985). Why are some problems hard? Evidence from tower of Hanoi. Cognitive Psychology, 17(2), 248\u2013294.","journal-title":"Cognitive Psychology"},{"issue":"4","key":"252_CR52","doi-asserted-by":"publisher","first-page":"978","DOI":"10.3758\/s13428-012-0210-4","volume":"44","author":"V Kuperman","year":"2012","unstructured":"Kuperman, V., Stadthagen-Gonzalez, H., & Brysbaert, M. (2012). Age-of-acquisition ratings for 30,000 english words. Behavior Research Methods, 44(4), 978\u2013990.","journal-title":"Behavior Research Methods"},{"issue":"1","key":"252_CR53","doi-asserted-by":"publisher","first-page":"121","DOI":"10.1007\/s40593-019-00186-y","volume":"30","author":"G Kurdi","year":"2020","unstructured":"Kurdi, G., Leo, J., Parsia, B., Sattler, U., & Al-Emari, S. (2020). A systematic review of automatic question generation for educational purposes. International Journal of Artificial Intelligence in Education, 30(1), 121\u2013204.","journal-title":"International Journal of Artificial Intelligence in Education"},{"issue":"2","key":"252_CR54","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1007\/s40593-018-00172-w","volume":"29","author":"J Leo","year":"2019","unstructured":"Leo, J., Kurdi, G., Matentzoglu, N., Parsia, B., Sattler, U., Forge, S., Donato, G., & Dowling, W. (2019). Ontology-based generation of medical, multi-term MCQS. International Journal of Artificial Intelligence in Education, 29(2), 145\u2013188.","journal-title":"International Journal of Artificial Intelligence in Education"},{"key":"252_CR55","doi-asserted-by":"crossref","unstructured":"Lin, C., Liu, D., Pang, W., & Apeh, E. (2015). Automatically predicting quiz difficulty level using similarity measures. In Proceedings of international conference on knowledge capture (pp. 1\u20138).","DOI":"10.1145\/2815833.2815842"},{"key":"252_CR56","doi-asserted-by":"crossref","unstructured":"Linehan, C., Bellord, G., Kirman, B., Morford, Z. H., & Roche, B. (2014). Learning curves: Analysing pace and challenge in four successful puzzle games. In Proceedings of computer-human interaction in play (pp. 181\u2013190). ACM.","DOI":"10.1145\/2658537.2658695"},{"issue":"6","key":"252_CR57","doi-asserted-by":"publisher","first-page":"553","DOI":"10.1016\/j.ergon.2012.09.001","volume":"42","author":"P Liu","year":"2012","unstructured":"Liu, P., & Li, Z. (2012). Task complexity: A review and conceptualization framework. International Journal of Industrial Ergonomics, 42(6), 553\u2013568.","journal-title":"International Journal of Industrial Ergonomics"},{"issue":"2","key":"252_CR58","doi-asserted-by":"publisher","first-page":"168","DOI":"10.1006\/cogp.1996.0016","volume":"31","author":"MC Lovett","year":"1996","unstructured":"Lovett, M. C., & Anderson, J. R. (1996). History of success and current context in problem solving: Combined influences on operator selection. Cognitive Psychology, 31(2), 168\u2013217.","journal-title":"Cognitive Psychology"},{"issue":"6","key":"252_CR59","doi-asserted-by":"publisher","first-page":"i","DOI":"10.1037\/h0093502","volume":"54","author":"AS Luchins","year":"1942","unstructured":"Luchins, A. S. (1942). Mechanization in problem solving: The effect of einstellung. Psychological Monographs, 54(6), i.","journal-title":"Psychological Monographs"},{"issue":"3","key":"252_CR60","doi-asserted-by":"publisher","first-page":"235","DOI":"10.1002\/rrq.019","volume":"47","author":"HA Mesmer","year":"2012","unstructured":"Mesmer, H. A., Cunningham, J. W., & Hiebert, E. H. (2012). Toward a theoretical model of text complexity for the early grades: Learning from the past, anticipating the future. Reading Research Quarterly, 47(3), 235\u2013258.","journal-title":"Reading Research Quarterly"},{"key":"252_CR61","unstructured":"Mikolov, T., Sutskever, I., Chen, K., Corrado, G. S., & Dean, J. (2013). Distributed representations of words and phrases and their compositionality. In Advances in neural information processing systems (pp. 3111\u20133119)."},{"key":"252_CR62","unstructured":"Miller, G. A. (1998). WordNet: An electronic lexical database. MIT Press."},{"key":"252_CR63","unstructured":"Milton, J. (2010). The development of vocabulary breadth across the CEFR levels. In Communicative proficiency and linguistic development: Intersections between SLA and language testing research, pp. 211\u2013232."},{"key":"252_CR64","doi-asserted-by":"crossref","unstructured":"Mitkov, R., Ha, L. A., Varga, A., & Rello, L. (2009). Semantic similarity of distractors in multiple-choice tests: Extrinsic evaluation. In Proceedings of the workshop on geometrical models of natural language semantics (pp. 49\u201356). Association for Computational Linguistics.","DOI":"10.3115\/1705415.1705422"},{"key":"252_CR65","doi-asserted-by":"crossref","unstructured":"Murray R. C., Ritter S., Nixon T., Schwiebert R., Hausmann R. G., Towle B., Fancsali S. E., & Vuong A. (2013). Revealing the learning in learning curves. In Proceedings of Artificial Intelligence in Education, (pp. 473\u2013482). Springer.","DOI":"10.1007\/978-3-642-39112-5_48"},{"key":"252_CR66","unstructured":"Nixon, T., Fancsali, S., & Ritter, S. (2013). The complex dynamics of aggregate learning curves. In Proceedings of educational data mining (pp. 338\u2013339)."},{"key":"252_CR67","doi-asserted-by":"crossref","unstructured":"Nuthong, S., & Witosurapot, S. (2017). Enabling fine granularity of difficulty ranking measure for automatic quiz generation. In Proceedings of information technology and electrical engineering (pp. 1\u20136). IEEE.","DOI":"10.1109\/ICITEED.2017.8250492"},{"key":"252_CR68","doi-asserted-by":"crossref","unstructured":"Pandarova, I., Schmidt, T., Hartig, J., Boubekki, A., Jones, R. D., & Brefeld, U. (2019). Predicting the difficulty of exercise items for dynamic difficulty adaptation in adaptive language tutoring. International Journal of Artificial Intelligence in Education, 1\u201326.","DOI":"10.1007\/s40593-019-00180-4"},{"key":"252_CR69","unstructured":"Papasalouros, A., Kanaris, K., & Kotis, K. (2008). Automatic generation of multiple choice questions from domain ontologies. e-Learning, 427\u2013434."},{"key":"252_CR70","unstructured":"Pel\u00e1nek, R. (2014). Difficulty rating of sudoku puzzles: An overview and evaluation. arXiv preprint arXiv:1403.7373."},{"key":"252_CR71","doi-asserted-by":"publisher","first-page":"169","DOI":"10.1016\/j.compedu.2016.03.017","volume":"98","author":"R Pel\u00e1nek","year":"2016","unstructured":"Pel\u00e1nek, R. (2016). Applications of the elo rating system in adaptive educational systems. Computers & Education, 98, 169\u2013179.","journal-title":"Computers & Education"},{"issue":"3","key":"252_CR72","doi-asserted-by":"publisher","first-page":"313","DOI":"10.1007\/s11257-017-9193-2","volume":"27","author":"R Pel\u00e1nek","year":"2017","unstructured":"Pel\u00e1nek, R. (2017). Bayesian knowledge tracing, logistic models, and beyond: An overview of learner modeling techniques. User Modeling and User-Adapted Interaction, 27(3), 313\u2013350.","journal-title":"User Modeling and User-Adapted Interaction"},{"key":"252_CR73","doi-asserted-by":"publisher","first-page":"207","DOI":"10.1007\/s11257-018-9204-y","volume":"28","author":"R Pel\u00e1nek","year":"2018","unstructured":"Pel\u00e1nek, R. (2018). The details matter: Methodological nuances in the evaluation of student models. User Modeling and User-Adapted Interaction, 28, 207\u2013235.","journal-title":"User Modeling and User-Adapted Interaction"},{"issue":"4","key":"252_CR74","doi-asserted-by":"publisher","first-page":"493","DOI":"10.1007\/s40593-015-0048-x","volume":"25","author":"R Pel\u00e1nek","year":"2015","unstructured":"Pel\u00e1nek, R., & Jaru\u0161ek, P. (2015). Student modeling based on problem solving times. International Journal of Artificial Intelligence in Education, 25(4), 493\u2013519.","journal-title":"International Journal of Artificial Intelligence in Education"},{"issue":"1","key":"252_CR75","doi-asserted-by":"publisher","first-page":"89","DOI":"10.1007\/s11257-016-9185-7","volume":"27","author":"R Pel\u00e1nek","year":"2017","unstructured":"Pel\u00e1nek, R., Papou\u0161ek, J., \u0158ih\u00e1k, J., Stanislav, V., & Ni\u017enan, J. (2017). Elo-based learner modeling for the adaptive practice of facts. User Modeling and User-Adapted Interaction, 27(1), 89\u2013118.","journal-title":"User Modeling and User-Adapted Interaction"},{"key":"252_CR76","unstructured":"Polozov, O., O\u2019Rourke, E., Smith, A. M., Zettlemoyer, L., Gulwani, S., & Popovi\u0107, Z. (2015). Personalized mathematical word problem generation. In Proceedings of international joint conference on artificial intelligence."},{"issue":"1","key":"252_CR77","doi-asserted-by":"publisher","first-page":"41","DOI":"10.1016\/S0160-2896(01)00067-8","volume":"30","author":"R Primi","year":"2001","unstructured":"Primi, R. (2001). Complexity of geometric inductive reasoning tasks: Contribution to the understanding of fluid intelligence. Intelligence, 30(1), 41\u201370.","journal-title":"Intelligence"},{"key":"252_CR78","doi-asserted-by":"crossref","unstructured":"Robertson S. (2004). Understanding inverse document frequency: On theoretical arguments for IDF. Journal of Documentation.","DOI":"10.1108\/00220410410560582"},{"issue":"1","key":"252_CR79","doi-asserted-by":"publisher","first-page":"27","DOI":"10.1093\/applin\/22.1.27","volume":"22","author":"P Robinson","year":"2001","unstructured":"Robinson, P. (2001). Task complexity, task difficulty, and task production: Exploring interactions in a componential framework. Applied Linguistics, 22(1), 27\u201357.","journal-title":"Applied Linguistics"},{"key":"252_CR80","unstructured":"Rosa, K. D., & Eskenazi, M. (2011). Effect of word complexity on l2 vocabulary learning. In Proceedings of workshop on innovative use of NLP for building educational applications (pp. 76\u201380). Association for Computational Linguistics."},{"key":"252_CR81","unstructured":"Sao Pedro, M., Baker, R., & Gobert, J. (2013). Incorporating scaffolding and tutor context into bayesian knowledge tracing to predict inquiry skill acquisition. In Educational Data Mining, 2013."},{"key":"252_CR82","unstructured":"Scheiter, K., & Gerjets, P. (2002). The impact of problem order: Sequencing problems as a strategy for improving one\u2019s performance. Proceedings of the Annual Meeting of the Cognitive Science Society, vol. 24."},{"key":"252_CR83","unstructured":"Schwarz, N., & Sudman, S. (2012). Context effects in social and psychological research. Springer Science & Business Media."},{"key":"252_CR84","doi-asserted-by":"crossref","unstructured":"Seyler, D., Yahya, M., & Berberich, K. (2017). Knowledge questions from knowledge graphs. Proceedings of theory of information retrieval, pp. 11\u201318.","DOI":"10.1145\/3121050.3121073"},{"key":"252_CR85","unstructured":"Sheard, J., Carbone, A., Chinn, D., Clear, T., Corney, M., D\u2019Souza, D., Fenwick, J., Harland, J., Laakso, M.-J., Teague, D., et al. (2013). How difficult are exams?: A framework for assessing the complexity of introductory programming exams. In Proceedings of australasian computing education conference (vol. 136, pp. 145\u2013154). Australian Computer Society, Inc."},{"key":"252_CR86","unstructured":"Sheehan, K. M., Kostin, I., & Futagi, Y. (2008). When do standard approaches for measuring vocabulary difficulty, syntactic complexity and referential cohesion yield biased estimates of text difficulty. In Proceedings of annual conference of the cognitive science society."},{"issue":"2","key":"252_CR87","doi-asserted-by":"publisher","first-page":"184","DOI":"10.1086\/678294","volume":"115","author":"KM Sheehan","year":"2014","unstructured":"Sheehan, K. M., Kostin, I., Napolitano, D., & Flor, M. (2014). The textevaluator tool: Helping teachers and test developers select texts for use in instruction and assessment. The Elementary School Journal, 115(2), 184\u2013209.","journal-title":"The Elementary School Journal"},{"issue":"5","key":"252_CR88","doi-asserted-by":"publisher","first-page":"1097","DOI":"10.1111\/bjet.12338","volume":"46","author":"GN Sohsah","year":"2015","unstructured":"Sohsah, G. N., \u00dcnal, M. E., & G\u00fczey, O. (2015). Classification of word levels with usage frequency, expert opinions and machine learning. British Journal of Educational Technology, 46(5), 1097\u20131101.","journal-title":"British Journal of Educational Technology"},{"key":"252_CR89","doi-asserted-by":"crossref","unstructured":"Susanti, Y., Nishikawa, H., Tokunaga, T., & Obari, H. (2016). Item difficulty analysis of english vocabulary questions. In Proceedings of conference on computer supported education (pp. 267\u2013274).","DOI":"10.5220\/0005775502670274"},{"issue":"6","key":"252_CR90","doi-asserted-by":"publisher","first-page":"837","DOI":"10.1002\/acp.1598","volume":"24","author":"K Taylor","year":"2010","unstructured":"Taylor, K., & Rohrer, D. (2010). The effects of interleaved practice. Applied Cognitive Psychology, 24(6), 837\u2013848.","journal-title":"Applied Cognitive Psychology"},{"key":"252_CR91","doi-asserted-by":"crossref","unstructured":"Thompson, B. (1984). Canonical correlation analysis: Uses and interpretation, number 47. Sage.","DOI":"10.4135\/9781412983570"},{"key":"252_CR92","unstructured":"Thompson, E., Luxton-Reilly, A., Whalley, J. L., Hu, M., & Robbins, P. (2008). Bloom\u2019s taxonomy for cs assessment. In Proceedings of Australasian computing education (pp. 155\u2013161). Australian Computer Society, Inc."},{"issue":"3","key":"252_CR93","doi-asserted-by":"publisher","first-page":"172","DOI":"10.1109\/TCIAIG.2011.2148116","volume":"3","author":"J Togelius","year":"2011","unstructured":"Togelius, J., Yannakakis, G. N., Stanley, K. O., & Browne, C. (2011). Search-based procedural content generation: A taxonomy and survey. IEEE Transactions on Computational Intelligence and AI in Games, 3(3), 172\u2013186.","journal-title":"IEEE Transactions on Computational Intelligence and AI in Games"},{"issue":"1","key":"252_CR94","first-page":"333","volume":"1","author":"T Uemura","year":"2004","unstructured":"Uemura, T., & Ishikawa, S. (2004). Jacet 8000 and Asia TEFL vocabulary initiative. Journal of Asia TEFL, 1(1), 333\u2013347.","journal-title":"Journal of Asia TEFL"},{"issue":"3","key":"252_CR95","doi-asserted-by":"publisher","first-page":"247","DOI":"10.1111\/j.1745-3984.2009.00080.x","volume":"46","author":"WJ Van Der Linden","year":"2009","unstructured":"Van Der Linden, W. J. (2009). Conceptual issues in response-time modeling. Journal of Educational Measurement, 46(3), 247\u2013272.","journal-title":"Journal of Educational Measurement"},{"key":"252_CR96","unstructured":"Van Merrienboer, J. J., & Krammer, H. P. (1990). The \u201ccompletion strategy\u201d in programming instruction: Theoretical and empirical support. In Research on instruction: Design and effects, pp. 45\u201361."},{"key":"252_CR97","unstructured":"Wang, K., & Su, Z. (2016). Dimensionally guided synthesis of mathematical word problems. In Proceedings of international joint conference on artificial intelligence (pp. 2661\u20132668)."},{"issue":"4","key":"252_CR98","doi-asserted-by":"publisher","first-page":"1183","DOI":"10.1016\/j.compedu.2011.11.020","volume":"58","author":"K Wauters","year":"2012","unstructured":"Wauters, K., Desmet, P., & Van Den Noortgate, W. (2012). Item difficulty estimation: An auspicious collaboration between data and judgment. Computers & Education, 58(4), 1183\u20131193.","journal-title":"Computers & Education"},{"key":"252_CR99","unstructured":"Webb, N. L. (1997). Criteria for alignment of expectations and assessments in mathematics and science education. In Number 6 in research monograph. Council of Chief State School Officers."},{"key":"252_CR100","unstructured":"Whalley, J., & Kasto, N. (2014). How difficult are novice code writing tasks?: A software metrics approach. In Proceedings of Australasian computing education conference (pp. 105\u2013112). Australian Computer Society, Inc."},{"key":"252_CR101","doi-asserted-by":"crossref","unstructured":"Yaneva, V., Baldwin, P., Mee, J., et al. (2019). Predicting the difficulty of multiple choice questions in a high-stakes medical exam. In Proceedings of workshop on innovative use of NLP for building educational applications, pp. 11\u201320.","DOI":"10.18653\/v1\/W19-4402"}],"container-title":["International Journal of Artificial Intelligence in Education"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s40593-021-00252-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s40593-021-00252-4\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s40593-021-00252-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,4]],"date-time":"2026-03-04T18:12:12Z","timestamp":1772647932000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s40593-021-00252-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,5,4]]},"references-count":101,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2022,3]]}},"alternative-id":["252"],"URL":"https:\/\/doi.org\/10.1007\/s40593-021-00252-4","relation":{},"ISSN":["1560-4292","1560-4306"],"issn-type":[{"value":"1560-4292","type":"print"},{"value":"1560-4306","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,5,4]]},"assertion":[{"value":"3 August 2020","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 March 2021","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 March 2021","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"4 May 2021","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}