{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,14]],"date-time":"2026-04-14T16:03:39Z","timestamp":1776182619428,"version":"3.50.1"},"reference-count":22,"publisher":"Allerton Press","issue":"7","license":[{"start":{"date-parts":[[2024,12,1]],"date-time":"2024-12-01T00:00:00Z","timestamp":1733011200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,12,1]],"date-time":"2024-12-01T00:00:00Z","timestamp":1733011200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Aut. Control Comp. Sci."],"published-print":{"date-parts":[[2024,12]]},"DOI":"10.3103\/s0146411624700329","type":"journal-article","created":{"date-parts":[[2025,2,12]],"date-time":"2025-02-12T14:48:52Z","timestamp":1739371732000},"page":"869-878","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":9,"title":["Text Classification by CEFR Levels Using Machine Learning Methods and the BERT Language Model"],"prefix":"10.3103","volume":"58","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6137-8643","authenticated-orcid":false,"given":"N. S.","family":"Lagutina","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1742-3240","authenticated-orcid":false,"given":"K. V.","family":"Lagutina","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0009-0003-1741-0604","authenticated-orcid":false,"given":"A. M.","family":"Brederman","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6757-9622","authenticated-orcid":false,"given":"N. N.","family":"Kasatkina","sequence":"additional","affiliation":[]}],"member":"1627","published-online":{"date-parts":[[2025,2,12]]},"reference":[{"key":"7753_CR1","doi-asserted-by":"publisher","first-page":"101258","DOI":"10.1016\/j.stueduc.2023.101258","volume":"77","author":"E. Del Gobbo","year":"2023","unstructured":"Del Gobbo, E., Guarino, A., Cafarelli, B., Grilli, L., and Limone, P., Automatic evaluation of open-ended questions for online learning. A systematic mapping, Stud. Educ. Eval., 2023, vol. 77, p. 101258. https:\/\/doi.org\/10.1016\/j.stueduc.2023.101258","journal-title":"Stud. Educ. Eval."},{"key":"7753_CR2","unstructured":"Galichev, N.V. and Shirogorodskaya, P.S., The problem of automatic scoring for complex constructs using open-ended tasks, XXI Mezhdunarodnaya nauchno-prakticheskaya konferenciya molodyh issledovatelej obrazovaniya (21st Int. Sci. and Pract. Conf. of Young Researchers of Education), Moscow: Mosk. Gos. Psikhol.-Pedagog. Univ., 2022, pp. 695\u2013697."},{"key":"7753_CR3","doi-asserted-by":"publisher","unstructured":"Adamova, L.E., Surikova, O.V., Bulatova, I.G., and Varlamov, O.O., Application of the mivar expert system to evaluate the complexity of texts, Izv. Kabardino-Balkarskogo Nauchn. Tsentra Ross. Akad. Nauk, 2021, no.\u00a02, pp. 11\u201329. https:\/\/doi.org\/10.35330\/1991-6639-2021-2-100-11-29","DOI":"10.35330\/1991-6639-2021-2-100-11-29"},{"key":"7753_CR4","doi-asserted-by":"publisher","first-page":"2495","DOI":"10.1007\/s10462-021-10068-2","volume":"55","author":"D. Ramesh","year":"2022","unstructured":"Ramesh, D. and Sanampudi, S.K., An automated essay scoring systems: A systematic literature review, Artif. Intell. Rev., 2022, vol. 55, no. 3, pp. 2495\u20132527. https:\/\/doi.org\/10.1007\/s10462-021-10068-2","journal-title":"Artif. Intell. Rev."},{"key":"7753_CR5","doi-asserted-by":"publisher","unstructured":"Yancey, K.P., Laflair, G., Verardi, A., and Burstein, J., Rating short L2 Essays on the CEFR scale with GPT\u20114, Proc. 18th Workshop on Innovative Use of NLP for Building Educational Applications (BEA 2023), Toronto, 2023, Kochmar, E., Burstein, J., Horbach, A., Laarmann-Quante, R., Madnani, N., Tack, A., Yaneva, V., Yuan, Zh., and Zesch, T., Eds., Association for Computational Linguistics, 2023, pp. 576\u2013584. https:\/\/doi.org\/10.18653\/v1\/2023.bea-1.49","DOI":"10.18653\/v1\/2023.bea-1.49"},{"key":"7753_CR6","doi-asserted-by":"publisher","first-page":"83","DOI":"10.3390\/info13020083","volume":"13","author":"A. Gasparetto","year":"2022","unstructured":"Gasparetto, A., Marcuzzo, M., Zangari, A., and Albarelli, A., A survey on text classification algorithms: From text to predictions, Information, 2022, vol. 13, no. 2, p. 83. https:\/\/doi.org\/10.3390\/info13020083","journal-title":"Information"},{"key":"7753_CR7","doi-asserted-by":"publisher","first-page":"5573","DOI":"10.1007\/s10639-021-10838-z","volume":"27","author":"V. Ramnarain-Seetohul","year":"2022","unstructured":"Ramnarain-Seetohul, V., Bassoo, V., and Rosunally, Ya., Similarity measures in automated essay scoring systems: A ten-year review, Educ. Inf. Technol., 2022, vol. 27, no. 4, pp. 5573\u20135604. https:\/\/doi.org\/10.1007\/s10639-021-10838-z","journal-title":"Educ. Inf. Technol."},{"key":"7753_CR8","doi-asserted-by":"publisher","unstructured":"Yang, P., Li, L., Luo, F., Liu, T., and Sun, X., Enhancing topic-to-essay generation with external commonsense knowledge, Proc. 57th Annu. Meeting of the Association for Computational Linguistics, Florence, 2019, Korhonen, A., Traum, D., and M\u00e0rquez, L., Eds., Association for Computational Linguistics, 2019, pp. 2002\u20132012. https:\/\/doi.org\/10.18653\/v1\/p19-1193","DOI":"10.18653\/v1\/p19-1193"},{"key":"7753_CR9","doi-asserted-by":"publisher","first-page":"405","DOI":"10.20310\/1810-0201-2023-28-2-405-414","volume":"28","author":"N.N. Mikheeva","year":"2023","unstructured":"Mikheeva, N.N. and Shulyndina, E.V., Features of training written internet-communication in a non-linguistic university, Vestn. Tambov. Univ., \n               Ser.: Gumanitarnye Nauki, 2023, vol. 28, no. 2, pp. 405\u2013414. https:\/\/doi.org\/10.20310\/1810-0201-2023-28-2-405-414","journal-title":"Ser.: Gumanitarnye Nauki"},{"key":"7753_CR10","doi-asserted-by":"publisher","unstructured":"Schmalz, V.J. and Brutti, A., Automatic assessment of english CEFR levels using BERT embeddings, Proc. Eighth Italian Conf. on Computational Linguistics CliC-It 2021, Fersini, E. et al., Eds., Accademia Univ. Press, 2021, pp. 293\u2013299. https:\/\/doi.org\/10.4000\/books.aaccademia.10828","DOI":"10.4000\/books.aaccademia.10828"},{"key":"7753_CR11","doi-asserted-by":"publisher","unstructured":"Arase, Yu., Uchida, S., and Kajiwara, T., CEFR-based sentence difficulty annotation and assessment, Proc. 2022 Conf. on Empirical Methods in Natural Language Processing, Abu Dhabi, 2022, Goldberg, Yo., Kozareva, Z., and Zhang, Yu., Eds., Association for Computational Linguistics, 2022, pp. 6206\u20136219. https:\/\/doi.org\/10.18653\/v1\/2022.emnlp-main.416","DOI":"10.18653\/v1\/2022.emnlp-main.416"},{"key":"7753_CR12","doi-asserted-by":"publisher","unstructured":"Jalota, R., Bourgonje, P., Van Sas, J., and Huang, H., Mitigating learnerese effects for CEFR classification, Proc. 17th Workshop on Innovative Use of NLP for Building Educational Applications (BEA 2022), Seattle, Wash., 2022, Kochmar, E., Burstein, J., Horbach, A., Laarmann-Quante, R., Madnani, N., Tack, A., Yaneva, V., Yuan, Zh., and Zesch, T., Eds., Association for Computational Linguistics, 2022, pp. 14\u201321. https:\/\/doi.org\/10.18653\/v1\/2022.bea-1.3","DOI":"10.18653\/v1\/2022.bea-1.3"},{"key":"7753_CR13","doi-asserted-by":"publisher","first-page":"130","DOI":"10.1017\/s095834402100029x","volume":"34","author":"T. Gaillat","year":"2022","unstructured":"Gaillat, T., Simpkin, A., Ballier, N., Stearns, B., Sousa, A., Bouy\u00e9, M., and Zarrouk, M., Predicting CEFR levels in learners of English: The use of microsystem criterial features in a machine learning approach, ReCALL, 2022, vol. 34, no. 2, pp. 130\u2013146. https:\/\/doi.org\/10.1017\/s095834402100029x","journal-title":"ReCALL"},{"key":"7753_CR14","volume-title":"Automated classification of written pro\u20acciency levels on the CEFR-scale through complexity contours and RNNs, Proc. 16th Workshop on Innovative Use of NLP for Building Educational Applications, Burstein, J., Horbach, A., Kochmar, E., Laarmann-Quante, R.","author":"E. Kerz","year":"2021","unstructured":"Kerz, E., Wiechmann, D., Qiao, Y., Tseng, E., and Str\u00f6bel, M., Automated classification of written pro\u20acciency levels on the CEFR-scale through complexity contours and RNNs, Proc. 16th Workshop on Innovative Use of NLP for Building Educational Applications, Burstein, J., Horbach, A., Kochmar, E., Laarmann-Quante, R., Leacock, C., Madnani, N., Pil\u00e1n, I., Yannakoudakis, H., and Zesch, T., Eds., Association for Computational Linguistics, 2021, pp. 199\u2013209. https:\/\/aclanthology.org\/2021.bea-1.21."},{"key":"7753_CR15","doi-asserted-by":"publisher","unstructured":"Yang, Yu. and Zhong, J., Automated essay scoring via example-based learning, Web Engineering. ICWE 2021, Brambilla, M., Chbeir, R., Frasincar, F., and Manolescu, I., Eds., Lecture Notes in Computer Science, vol.\u00a012706, Springer, 2021, pp. 201\u2013208. https:\/\/doi.org\/10.1007\/978-3-030-74296-6_16","DOI":"10.1007\/978-3-030-74296-6_16"},{"key":"7753_CR16","doi-asserted-by":"publisher","unstructured":"Mayfield, E. and Black, A.W., Should you finetune BERT for automated essay scoring?, Proc. Fifteenth Workshop on Innovative Use of NLP for Building Educational Applications, Seattle, Wash., 2020, Burstein, J., Kochmar, E., Leacock, C., Madnani, N., Pil\u00e1n, I., Yannakoudakis, H., and Zesch, T., Eds., Association for Computational Linguistics, 2020, pp. 151\u2013162. https:\/\/doi.org\/10.18653\/v1\/2020.bea-1.15","DOI":"10.18653\/v1\/2020.bea-1.15"},{"key":"7753_CR17","volume-title":"BERT embeddings for automatic readability assessment, Proc. Int. Conf. on Recent Advances in Natural Language Processing (RANLP 2021)","author":"J.M. Imperial","year":"2021","unstructured":"Imperial, J.M., BERT embeddings for automatic readability assessment, Proc. Int. Conf. on Recent Advances in Natural Language Processing (RANLP 2021), Mitkov, R. and Angelova, G., Eds., INCOMA, 2021, pp. 611\u2013618. https:\/\/aclanthology.org\/2021.ranlp-1.69."},{"key":"7753_CR18","doi-asserted-by":"publisher","unstructured":"Bryant, Ch., Felice, M., Andersen, \u00d8.E., and Briscoe, T., The BEA-2019 shared task on grammatical error correction, Proc. Fourteenth Workshop on Innovative Use of NLP for Building Educational Applications, Florence, 2019, Yannakoudakis, H., Kochmar, E., Leacock, C., Madnani, N., Pil\u00e1n, I., and Zesch, T., Eds., Association for Computational Linguistics, 2019, pp. 52\u201375. https:\/\/doi.org\/10.18653\/v1\/w19-4406","DOI":"10.18653\/v1\/w19-4406"},{"key":"7753_CR19","doi-asserted-by":"publisher","first-page":"866","DOI":"10.3103\/s0146411621070257","volume":"55","author":"K.V. Lagutina","year":"2021","unstructured":"Lagutina, K.V. and Manakhova, A.M., Automated search and analysis of the stylometric features that describe the style of the prose of 19th\u201321st centuries, Autom. Control Comput. Sci., 2021, vol. 55, no. 7, pp. 866\u2013876. https:\/\/doi.org\/10.3103\/s0146411621070257","journal-title":"Autom. Control Comput. Sci."},{"key":"7753_CR20","doi-asserted-by":"publisher","first-page":"260","DOI":"10.18255\/1818-1015-2021-3-260-279","volume":"28","author":"A.M. Manakhova","year":"2021","unstructured":"Manakhova, A.M. and Lagutina, N.S., Analysis of the impact of the stylometric characteristics of different levels for the verification of authors of the prose, Model. Anal. Inf. Syst., 2021, vol. 28, no. 3, pp. 260\u2013279. https:\/\/doi.org\/10.18255\/1818-1015-2021-3-260-279","journal-title":"Model. Anal. Inf. Syst."},{"key":"7753_CR21","doi-asserted-by":"publisher","unstructured":"Devlin, J., Chang, M.-W., Lee, K., and Toutanova, K., BERT: Pre-training of deep bidirectional transformers for language understanding, Proc. 2019 Conf. of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Minneapolis, Minn., 2019, Burstein, J., Doran, Ch., and Solorio, T., Eds., 2019, vol. 1, pp. 4171\u20134186. https:\/\/doi.org\/10.18653\/v1\/N19-1423","DOI":"10.18653\/v1\/N19-1423"},{"key":"7753_CR22","doi-asserted-by":"publisher","unstructured":"Sanh, V., Debut, L., Chaumond, J., and Wolf, T., DistilBERT, a distilled version of BERT: Smaller, faster, cheaper and lighter, arXiv Preprint, 2020. https:\/\/doi.org\/10.48550\/arXiv.1910.01108","DOI":"10.48550\/arXiv.1910.01108"}],"container-title":["Automatic Control and Computer Sciences"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.3103\/S0146411624700329.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.3103\/S0146411624700329","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.3103\/S0146411624700329.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,15]],"date-time":"2026-03-15T22:03:01Z","timestamp":1773612181000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.3103\/S0146411624700329"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,12]]},"references-count":22,"journal-issue":{"issue":"7","published-print":{"date-parts":[[2024,12]]}},"alternative-id":["7753"],"URL":"https:\/\/doi.org\/10.3103\/s0146411624700329","relation":{},"ISSN":["0146-4116","1558-108X"],"issn-type":[{"value":"0146-4116","type":"print"},{"value":"1558-108X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,12]]},"assertion":[{"value":"14 August 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 August 2023","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 August 2023","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"12 February 2025","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors of this work declare that they have no conflicts of interest.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"CONFLICT OF INTEREST"}}]}}