{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,15]],"date-time":"2026-05-15T14:51:01Z","timestamp":1778856661706,"version":"3.51.4"},"reference-count":38,"publisher":"MDPI AG","issue":"10","license":[{"start":{"date-parts":[[2023,10,14]],"date-time":"2023-10-14T00:00:00Z","timestamp":1697241600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100005802","name":"Ministry of Research, Innovation, and Digitalization","doi-asserted-by":"publisher","award":["344\/390020\/06.09.2021"],"award-info":[{"award-number":["344\/390020\/06.09.2021"]}],"id":[{"id":"10.13039\/501100005802","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100005802","name":"Ministry of Research, Innovation, and Digitalization","doi-asserted-by":"publisher","award":["62461\/03.06.2022"],"award-info":[{"award-number":["62461\/03.06.2022"]}],"id":[{"id":"10.13039\/501100005802","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100005802","name":"Ministry of Research, Innovation, and Digitalization","doi-asserted-by":"publisher","award":["R305A130124"],"award-info":[{"award-number":["R305A130124"]}],"id":[{"id":"10.13039\/501100005802","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100005802","name":"Ministry of Research, Innovation, and Digitalization","doi-asserted-by":"publisher","award":["R305A190063"],"award-info":[{"award-number":["R305A190063"]}],"id":[{"id":"10.13039\/501100005802","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100005802","name":"Ministry of Research, Innovation, and Digitalization","doi-asserted-by":"publisher","award":["N00014-20-1-2623"],"award-info":[{"award-number":["N00014-20-1-2623"]}],"id":[{"id":"10.13039\/501100005802","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100005802","name":"Ministry of Research, Innovation, and Digitalization","doi-asserted-by":"publisher","award":["NSF REC0241144"],"award-info":[{"award-number":["NSF REC0241144"]}],"id":[{"id":"10.13039\/501100005802","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100005802","name":"Ministry of Research, Innovation, and Digitalization","doi-asserted-by":"publisher","award":["IIS-0735682"],"award-info":[{"award-number":["IIS-0735682"]}],"id":[{"id":"10.13039\/501100005802","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Ministry of European Investments and Projects","award":["344\/390020\/06.09.2021"],"award-info":[{"award-number":["344\/390020\/06.09.2021"]}]},{"name":"Ministry of European Investments and Projects","award":["62461\/03.06.2022"],"award-info":[{"award-number":["62461\/03.06.2022"]}]},{"name":"Ministry of European Investments and Projects","award":["R305A130124"],"award-info":[{"award-number":["R305A130124"]}]},{"name":"Ministry of European Investments and Projects","award":["R305A190063"],"award-info":[{"award-number":["R305A190063"]}]},{"name":"Ministry of European Investments and Projects","award":["N00014-20-1-2623"],"award-info":[{"award-number":["N00014-20-1-2623"]}]},{"name":"Ministry of European Investments and Projects","award":["NSF REC0241144"],"award-info":[{"award-number":["NSF REC0241144"]}]},{"name":"Ministry of European Investments and Projects","award":["IIS-0735682"],"award-info":[{"award-number":["IIS-0735682"]}]},{"DOI":"10.13039\/100005246","name":"IES U.S. Department of Education","doi-asserted-by":"publisher","award":["344\/390020\/06.09.2021"],"award-info":[{"award-number":["344\/390020\/06.09.2021"]}],"id":[{"id":"10.13039\/100005246","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100005246","name":"IES U.S. Department of Education","doi-asserted-by":"publisher","award":["62461\/03.06.2022"],"award-info":[{"award-number":["62461\/03.06.2022"]}],"id":[{"id":"10.13039\/100005246","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100005246","name":"IES U.S. Department of Education","doi-asserted-by":"publisher","award":["R305A130124"],"award-info":[{"award-number":["R305A130124"]}],"id":[{"id":"10.13039\/100005246","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100005246","name":"IES U.S. Department of Education","doi-asserted-by":"publisher","award":["R305A190063"],"award-info":[{"award-number":["R305A190063"]}],"id":[{"id":"10.13039\/100005246","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100005246","name":"IES U.S. Department of Education","doi-asserted-by":"publisher","award":["N00014-20-1-2623"],"award-info":[{"award-number":["N00014-20-1-2623"]}],"id":[{"id":"10.13039\/100005246","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100005246","name":"IES U.S. Department of Education","doi-asserted-by":"publisher","award":["NSF REC0241144"],"award-info":[{"award-number":["NSF REC0241144"]}],"id":[{"id":"10.13039\/100005246","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100005246","name":"IES U.S. Department of Education","doi-asserted-by":"publisher","award":["IIS-0735682"],"award-info":[{"award-number":["IIS-0735682"]}],"id":[{"id":"10.13039\/100005246","id-type":"DOI","asserted-by":"publisher"}]},{"name":"the Office of Naval Research","award":["344\/390020\/06.09.2021"],"award-info":[{"award-number":["344\/390020\/06.09.2021"]}]},{"name":"the Office of Naval Research","award":["62461\/03.06.2022"],"award-info":[{"award-number":["62461\/03.06.2022"]}]},{"name":"the Office of Naval Research","award":["R305A130124"],"award-info":[{"award-number":["R305A130124"]}]},{"name":"the Office of Naval Research","award":["R305A190063"],"award-info":[{"award-number":["R305A190063"]}]},{"name":"the Office of Naval Research","award":["N00014-20-1-2623"],"award-info":[{"award-number":["N00014-20-1-2623"]}]},{"name":"the Office of Naval Research","award":["NSF REC0241144"],"award-info":[{"award-number":["NSF REC0241144"]}]},{"name":"the Office of Naval Research","award":["IIS-0735682"],"award-info":[{"award-number":["IIS-0735682"]}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["344\/390020\/06.09.2021"],"award-info":[{"award-number":["344\/390020\/06.09.2021"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["62461\/03.06.2022"],"award-info":[{"award-number":["62461\/03.06.2022"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["R305A130124"],"award-info":[{"award-number":["R305A130124"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["R305A190063"],"award-info":[{"award-number":["R305A190063"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["N00014-20-1-2623"],"award-info":[{"award-number":["N00014-20-1-2623"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["NSF REC0241144"],"award-info":[{"award-number":["NSF REC0241144"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["IIS-0735682"],"award-info":[{"award-number":["IIS-0735682"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Information"],"abstract":"<jats:p>Text comprehension is an essential skill in today\u2019s information-rich world, and self-explanation practice helps students improve their understanding of complex texts. This study was centered on leveraging open-source Large Language Models (LLMs), specifically FLAN-T5, to automatically assess the comprehension strategies employed by readers while understanding Science, Technology, Engineering, and Mathematics (STEM) texts. The experiments relied on a corpus of three datasets (N = 11,833) with self-explanations annotated on 4 dimensions: 3 comprehension strategies (i.e., bridging, elaboration, and paraphrasing) and overall quality. Besides FLAN-T5, we also considered GPT3.5-turbo to establish a stronger baseline. Our experiments indicated that the performance improved with fine-tuning, having a larger LLM model, and providing examples via the prompt. Our best model considered a pretrained FLAN-T5 XXL model and obtained a weighted F1-score of 0.721, surpassing the 0.699 F1-score previously obtained using smaller models (i.e., RoBERTa).<\/jats:p>","DOI":"10.3390\/info14100567","type":"journal-article","created":{"date-parts":[[2023,10,14]],"date-time":"2023-10-14T14:59:59Z","timestamp":1697295599000},"page":"567","update-policy":"https:\/\/doi.org\/10.3390\/mdpi_crossmark_policy","source":"Crossref","is-referenced-by-count":14,"title":["Automated Assessment of Comprehension Strategies from Self-Explanations Using LLMs"],"prefix":"10.3390","volume":"14","author":[{"given":"Bogdan","family":"Nicula","sequence":"first","affiliation":[{"name":"Computer Science and Engineering Department, National University of Science and Technology POLITEHNICA of Bucharest, 313 Splaiul Independentei, 060042 Bucharest, Romania"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4815-9227","authenticated-orcid":false,"given":"Mihai","family":"Dascalu","sequence":"additional","affiliation":[{"name":"Computer Science and Engineering Department, National University of Science and Technology POLITEHNICA of Bucharest, 313 Splaiul Independentei, 060042 Bucharest, Romania"},{"name":"Academy of Romanian Scientists, Str. Ilfov, Nr. 3, 050044 Bucharest, Romania"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5072-8636","authenticated-orcid":false,"given":"Tracy","family":"Arner","sequence":"additional","affiliation":[{"name":"Department of Psychology, Arizona State University, P.O. Box 871104, Tempe, AZ 85287, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1393-2416","authenticated-orcid":false,"given":"Renu","family":"Balyan","sequence":"additional","affiliation":[{"name":"Math\/CIS Department, SUNY Old Westbury, Old Westbury, NY 11568, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5869-1420","authenticated-orcid":false,"given":"Danielle S.","family":"McNamara","sequence":"additional","affiliation":[{"name":"Department of Psychology, Arizona State University, P.O. Box 871104, Tempe, AZ 85287, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"1968","published-online":{"date-parts":[[2023,10,14]]},"reference":[{"key":"ref_1","doi-asserted-by":"crossref","first-page":"479","DOI":"10.1080\/0163853X.2015.1101328","article-title":"Self-explanation and reading strategy training (SERT) improves low-knowledge students\u2019 science course performance","volume":"54","author":"McNamara","year":"2017","journal-title":"Discourse Process."},{"key":"ref_2","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1207\/s15326950dp3801_1","article-title":"SERT: Self-explanation reading training","volume":"38","author":"McNamara","year":"2004","journal-title":"Discourse Process."},{"key":"ref_3","doi-asserted-by":"crossref","first-page":"988","DOI":"10.1111\/j.1553-2712.2008.00227.x","article-title":"Deliberate practice and acquisition of expert performance: A general overview","volume":"15","year":"2008","journal-title":"Acad. Emerg. Med."},{"key":"ref_4","doi-asserted-by":"crossref","first-page":"2239","DOI":"10.1080\/10447318.2022.2114143","article-title":"iSTART: Adaptive Comprehension Strategy Training and Stealth Literacy Assessment","volume":"39","author":"McNamara","year":"2023","journal-title":"Int. J.-Hum.-Comput. Interact."},{"key":"ref_5","doi-asserted-by":"crossref","unstructured":"McNamara, D.S., O\u2019Reilly, T., Rowe, M., Boonthum, C., and Levinstein, I.B. (2007). Reading Comprehension Strategies: Theories, Interventions, and Technologies, Lawrence Erlbaum Associates Publishers.","DOI":"10.4324\/9780203810033"},{"key":"ref_6","unstructured":"Jurafsky, D., and Martin, J.H. (2009). Speech and Language Processing: An Introduction to Natural Language Processing, Computational Linguistics, and Speech Recognition, Pearson Prentice Hall."},{"key":"ref_7","doi-asserted-by":"crossref","unstructured":"Nicula, B., Panaite, M., Arner, T., Balyan, R., Dascalu, M., and McNamara, D.S. (2023, January 3\u20137). Automated Assessment of Comprehension Strategies from Self-explanations Using Transformers and Multi-task Learning. Proceedings of the International Conference on Artificial Intelligence in Education, Tokyo, Japan.","DOI":"10.1007\/978-3-031-36336-8_107"},{"key":"ref_8","unstructured":"OpenAI (2023, October 05). Introducing ChatGPT. Available online: https:\/\/openai.com\/blog\/chatgpt."},{"key":"ref_9","unstructured":"Pichai, S. (2023, October 05). An Important Next Step on Our AI Journey. Available online: https:\/\/blog.google\/technology\/ai\/bard-google-ai-search-updates\/."},{"key":"ref_10","unstructured":"Wei, J., Tay, Y., Bommasani, R., Raffel, C., Zoph, B., Borgeaud, S., Yogatama, D., Bosma, M., Zhou, D., and Metzler, D. (2022). Emergent Abilities of Large Language Models. arXiv."},{"key":"ref_11","first-page":"5998","article-title":"Attention is all you need","volume":"Volume 30","author":"Vaswani","year":"2017","journal-title":"Proceedings of the Advances in Neural Information Processing Systems"},{"key":"ref_12","unstructured":"Devlin, J., Chang, M.W., Lee, K., and Toutanova, K. (2019, January 2\u20137). BERT: Pretraining of Deep Bidirectional Transformers for Language Understanding. Proceedings of the 2019 Conference of the NAACL, Minneapolis, MN, USA."},{"key":"ref_13","unstructured":"Liu, Y., Ott, M., Goyal, N., Du, J., Joshi, M., Chen, D., Levy, O., Lewis, M., Zettlemoyer, L., and Stoyanov, V. (2019). RoBERTa: A Robustly Optimized BERT Pretraining Approach. arXiv."},{"key":"ref_14","unstructured":"Brown, T.B., Mann, B., Ryder, N., Subbiah, M., Kaplan, J., Dhariwal, P., Neelakantan, A., Shyam, P., Sastry, G., and Askell, A. (2020). Language Models are Few-Shot Learners. arXiv."},{"key":"ref_15","unstructured":"OpenAI (2023). GPT-4 Technical Report. arXiv."},{"key":"ref_16","first-page":"1","article-title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","volume":"21","author":"Raffel","year":"2020","journal-title":"J. Mach. Learn. Res."},{"key":"ref_17","unstructured":"Chung, H.W., Hou, L., Longpre, S., Zoph, B., Tay, Y., Fedus, W., Li, E., Wang, X., Dehghani, M., and Brahma, S. (2022). Scaling Instruction-Finetuned Language Models. arXiv."},{"key":"ref_18","doi-asserted-by":"crossref","unstructured":"Chiesurin, S., Dimakopoulos, D., Cabezudo, M.A.S., Eshghi, A., Papaioannou, I., Rieser, V., and Konstas, I. (2023). The Dangers of trusting Stochastic Parrots: Faithfulness and Trust in Open-domain Conversational Question Answering. arXiv.","DOI":"10.18653\/v1\/2023.findings-acl.60"},{"key":"ref_19","unstructured":"Perez, F., and Ribeiro, I. (2022). Ignore Previous Prompt: Attack Techniques For Language Models. arXiv."},{"key":"ref_20","doi-asserted-by":"crossref","unstructured":"Wang, A., Singh, A., Michael, J., Hill, F., Levy, O., and Bowman, S.R. (2019). GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding. arXiv.","DOI":"10.18653\/v1\/W18-5446"},{"key":"ref_21","unstructured":"Wang, A., Pruksachatkun, Y., Nangia, N., Singh, A., Michael, J., Hill, F., Levy, O., and Bowman, S.R. (2020). SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems. arXiv."},{"key":"ref_22","unstructured":"Chowdhery, A., Narang, S., Devlin, J., Bosma, M., Mishra, G., Roberts, A., Barham, P., Chung, H.W., Sutton, C., and Gehrmann, S. (2022). PaLM: Scaling Language Modeling with Pathways. arXiv."},{"key":"ref_23","first-page":"8","article-title":"Improving Language Understanding with Unsupervised Learning","volume":"1","author":"Radford","year":"2018","journal-title":"OpenAI Blog"},{"key":"ref_24","first-page":"9","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford","year":"2019","journal-title":"OpenAI Blog"},{"key":"ref_25","unstructured":"Guyon, I., Luxburg, U.V., Bengio, S., Wallach, H., Fergus, R., Vishwanathan, S., and Garnett, R. Deep Reinforcement Learning from Human Preferences. Proceedings of the Advances in Neural Information Processing Systems, 31st Conference on Neural Information Processing Systems (NIPS 2017), Long Beach, CA, USA, 4\u20139 December 2017."},{"key":"ref_26","doi-asserted-by":"crossref","first-page":"337","DOI":"10.1080\/0163853X.2023.2225757","article-title":"Anchoring your bridge: The importance of paraphrasing to inference making in self-explanations","volume":"60","author":"McNamara","year":"2023","journal-title":"Discourse Process."},{"key":"ref_27","unstructured":"Liu, P., Yuan, W., Fu, J., Jiang, Z., Hayashi, H., and Neubig, G. (2021). Pretrain, Prompt, and Predict: A Systematic Survey of Prompting Methods in Natural Language Processing. arXiv."},{"key":"ref_28","unstructured":"OpenAI (2023, August 10). Chat Completions API Documentation. Available online: https:\/\/platform.openai.com\/docs\/guides\/gpt\/chat-completions-api."},{"key":"ref_29","unstructured":"Liu, X., Zheng, Y., Du, Z., Ding, M., Qian, Y., Yang, Z., and Tang, J. (2021). GPT Understands, Too. arXiv."},{"key":"ref_30","doi-asserted-by":"crossref","unstructured":"Li, X.L., and Liang, P. (2021). Prefix-Tuning: Optimizing Continuous Prompts for Generation. arXiv.","DOI":"10.18653\/v1\/2021.acl-long.353"},{"key":"ref_31","unstructured":"Hu, E.J., Shen, Y., Wallis, P., Allen-Zhu, Z., Li, Y., Wang, S., Wang, L., and Chen, W. (2021). LoRA: Low-Rank Adaptation of Large Language Models. arXiv."},{"key":"ref_32","unstructured":"Smith, S.L., Kindermans, P., and Le, Q.V. (2017). Don\u2019t Decay the Learning Rate, Increase the Batch Size. arXiv."},{"key":"ref_33","doi-asserted-by":"crossref","unstructured":"He, Y., Zhang, X., and Sun, J. (2017). Channel Pruning for Accelerating Very Deep Neural Networks. arXiv.","DOI":"10.1109\/ICCV.2017.155"},{"key":"ref_34","doi-asserted-by":"crossref","first-page":"119391","DOI":"10.1016\/j.eswa.2022.119391","article-title":"Representation and compression of Residual Neural Networks through a multilayer network based approach","volume":"215","author":"Amelio","year":"2023","journal-title":"Expert Syst. Appl."},{"key":"ref_35","unstructured":"Huang, K., Ni, B., and Yang, X. (February, January 27). Efficient quantization for neural networks with binary weights and low bitwidth activations. Proceedings of the AAAI Conference on Artificial Intelligence, Honolulu, HI, USA."},{"key":"ref_36","first-page":"6869","article-title":"Quantized neural networks: Training neural networks with low precision weights and activations","volume":"18","author":"Hubara","year":"2017","journal-title":"J. Mach. Learn. Res."},{"key":"ref_37","unstructured":"Han, S., Mao, H., and Dally, W.J. (2015). Deep compression: Compressing deep neural networks with pruning, trained quantization and huffman coding. arXiv."},{"key":"ref_38","unstructured":"Ma, X., Fang, G., and Wang, X. (2023). LLM-Pruner: On the Structural Pruning of Large Language Models. arXiv."}],"container-title":["Information"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.mdpi.com\/2078-2489\/14\/10\/567\/pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,10,10]],"date-time":"2025-10-10T21:07:03Z","timestamp":1760130423000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.mdpi.com\/2078-2489\/14\/10\/567"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,10,14]]},"references-count":38,"journal-issue":{"issue":"10","published-online":{"date-parts":[[2023,10]]}},"alternative-id":["info14100567"],"URL":"https:\/\/doi.org\/10.3390\/info14100567","relation":{},"ISSN":["2078-2489"],"issn-type":[{"value":"2078-2489","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,10,14]]}}}