{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,19]],"date-time":"2026-06-19T15:50:39Z","timestamp":1781884239818,"version":"3.54.5"},"reference-count":41,"publisher":"Elsevier BV","issue":"2","license":[{"start":{"date-parts":[[2024,6,1]],"date-time":"2024-06-01T00:00:00Z","timestamp":1717200000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2024,6,1]],"date-time":"2024-06-01T00:00:00Z","timestamp":1717200000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,6,15]],"date-time":"2026-06-15T00:00:00Z","timestamp":1781481600000},"content-version":"vor","delay-in-days":744,"URL":"http:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100009244","name":"Stockholm University","doi-asserted-by":"crossref","id":[{"id":"10.13039\/501100009244","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["International Journal of Artificial Intelligence in Education"],"published-print":{"date-parts":[[2024,6]]},"DOI":"10.1007\/s40593-022-00322-1","type":"journal-article","created":{"date-parts":[[2023,2,28]],"date-time":"2023-02-28T13:03:06Z","timestamp":1677589386000},"page":"247-273","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":38,"title":["Reducing Workload in Short Answer Grading Using Machine Learning"],"prefix":"10.1016","volume":"34","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6403-0020","authenticated-orcid":false,"given":"Rebecka","family":"Weegar","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Peter","family":"Idestam-Almquist","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1007\/s40593-022-00322-1_bib1","unstructured":"Anderson, L. W., Bloom, B. S., & et al. (2001). A taxonomy for learning, teaching, and assessing: A revision of Bloom\u2019s taxonomy of educational objectives. Longman."},{"key":"10.1007\/s40593-022-00322-1_bib2","doi-asserted-by":"crossref","unstructured":"Azad, S., Chen, B., Fowler, M., West, M., & Zilles, C. (2020). Strategies for deploying unreliable ai graders in high-transparency high-stakes exams. In: International Conference on Artificial Intelligence in Education. Springer, pp 16\u201328","DOI":"10.1007\/978-3-030-52237-7_2"},{"key":"10.1007\/s40593-022-00322-1_bib3","doi-asserted-by":"crossref","first-page":"391","DOI":"10.1162\/tacl_a_00236","article-title":"Powergrading: a clustering approach to amplify human effort for short answer grading","volume":"1","author":"Basu","year":"2013","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"10.1007\/s40593-022-00322-1_bib4","unstructured":"Bonthu, S. (2021). Automated short answer grading using deep learning: A survey. In: Machine Learning and Knowledge Extraction: 5th IFIP TC 5, TC 12, WG 8.4, WG 8.9, WG 12.9 International Cross-Domain Conference, CD-MAKE 2021, Virtual Event, August 17-20, 2021, Proceedings, Springer Nature, vol 12844, p 61"},{"issue":"1","key":"10.1007\/s40593-022-00322-1_bib5","doi-asserted-by":"crossref","first-page":"5","DOI":"10.1023\/A:1010933404324","article-title":"Random forests.","volume":"45","author":"Breiman","year":"2001","journal-title":"Machine learning"},{"key":"10.1007\/s40593-022-00322-1_bib6","doi-asserted-by":"crossref","unstructured":"Brooks, M., Basu, S., Jacobs, C., & Vanderwende, L. (2014). Divide and correct: using clusters to grade short answers at scale. In: Proceedings of the first ACM conference on Learning@ scale conference, pp 89\u201398","DOI":"10.1145\/2556325.2566243"},{"issue":"1","key":"10.1007\/s40593-022-00322-1_bib7","doi-asserted-by":"crossref","first-page":"60","DOI":"10.1007\/s40593-014-0026-8","article-title":"The eras and trends of automatic short answer grading","volume":"25","author":"Burrows","year":"2015","journal-title":"International Journal of Artificial Intelligence in Education"},{"issue":"4","key":"10.1007\/s40593-022-00322-1_bib8","doi-asserted-by":"crossref","first-page":"213","DOI":"10.1037\/h0026256","article-title":"Weighted kappa: nominal scale agreement provision for scaled disagreement or partial credit","volume":"70","author":"Cohen","year":"1968","journal-title":"Psychological bulletin"},{"key":"10.1007\/s40593-022-00322-1_bib9","unstructured":"Devlin, J., Chang, M. W., Lee, K., & Toutanova, K. (2018). Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv:1810.04805"},{"key":"10.1007\/s40593-022-00322-1_bib10","doi-asserted-by":"crossref","unstructured":"Filighera, A., Steuer, T., Rensing, C. (2020). Fooling automatic short answer grading systems. In: International Conference on Artificial Intelligence in Education. Springer, pp 177\u2013190","DOI":"10.1007\/978-3-030-52237-7_15"},{"key":"10.1007\/s40593-022-00322-1_bib11","doi-asserted-by":"crossref","unstructured":"Galhardi, L., de Souza, R. C. T., & Brancher, J. (2020). Automatic grading of portuguese short answers using a machine learning approach. In: Anais Estendidos do XVI Simp\u00f3sio Brasileiro de Sistemas de Informa\u00e7\u00e3o, SBC, pp 109\u2013124","DOI":"10.5753\/sbsi.2020.13133"},{"key":"10.1007\/s40593-022-00322-1_bib12","doi-asserted-by":"crossref","unstructured":"Geigle, C., Zhai, C., & Ferguson, D. C. (2016). An exploration of automated grading of complex assignments. In: Proceedings of the Third (2016) ACM Conference on Learning@ Scale, pp 351\u2013360","DOI":"10.1145\/2876034.2876049"},{"key":"10.1007\/s40593-022-00322-1_bib13","doi-asserted-by":"crossref","unstructured":"Gomaa, W. H., & Fahmy, A. A. (2014) Arabic short answer scoring with effective feedback for students. International Journal of Computer Applications 86(2)","DOI":"10.5120\/14961-3177"},{"key":"10.1007\/s40593-022-00322-1_bib14","doi-asserted-by":"crossref","unstructured":"Horbach, A., & Palmer, A. (2016) Investigating active learning for short-answer scoring. In: Proceedings of the 11th workshop on innovative use of NLP for building educational applications, pp 301\u2013311","DOI":"10.18653\/v1\/W16-0535"},{"key":"10.1007\/s40593-022-00322-1_bib15","doi-asserted-by":"crossref","unstructured":"Horbach, A., & Pinkal, M. (2018). Semi-supervised clustering for short answer scoring. In: Proceedings of the Eleventh International Conference on Language Resources and Evaluation (LREC 2018), pp 4066\u20134071","DOI":"10.63317\/45z7js2bo39m"},{"key":"10.1007\/s40593-022-00322-1_bib16","doi-asserted-by":"crossref","unstructured":"Horbach, A., & Zesch, T. (2019). The influence of variance in learner answers on automatic content scoring. In: Frontiers in Education, Frontiers, vol 4, p 28","DOI":"10.3389\/feduc.2019.00028"},{"key":"10.1007\/s40593-022-00322-1_bib17","doi-asserted-by":"crossref","unstructured":"Horbach, A., Palmer, A., & Wolska, M. (2014). Finding a tradeoff between accuracy and rater\u2019s workload in grading clustered short answers. In: LREC, Citeseer, pp 588\u2013595","DOI":"10.63317\/4voiu6s3b3de"},{"issue":"3","key":"10.1007\/s40593-022-00322-1_bib18","doi-asserted-by":"crossref","first-page":"90","DOI":"10.1109\/MCSE.2007.55","article-title":"Matplotlib: A 2d graphics environment","volume":"9","author":"Hunter","year":"2007","journal-title":"Computing in Science & Engineering"},{"key":"10.1007\/s40593-022-00322-1_bib19","doi-asserted-by":"crossref","unstructured":"Kang, J., Ryu, K. R., Kwon, H. C. (2004) Using cluster-based sampling to select initial training set for active learning in text classification. In: Pacific-Asia conference on knowledge discovery and data mining, Springer, pp 384\u2013388","DOI":"10.1007\/978-3-540-24775-3_46"},{"key":"10.1007\/s40593-022-00322-1_bib20","first-page":"6300","article-title":"Automated essay scoring: A survey of the state of the art","volume":"19","author":"Ke","year":"2019","journal-title":"IJCAI"},{"key":"10.1007\/s40593-022-00322-1_bib21","unstructured":"Kishaan, J., Muthuraja, M., Nair, D., & Pl\u00f6ger, P. G. (2020). Using active learning for assisted short answer grading. In: ICML 2020 Workshop on Real World Experiment Design and Active Learning"},{"key":"10.1007\/s40593-022-00322-1_bib22","doi-asserted-by":"crossref","unstructured":"Kulkarni, C. E., Socher, R., & Bernstein, M. S., Klemmer, S. R. (2014). Scaling short-answer grading by combining peer assessment with algorithmic scoring. In: Proceedings of the first ACM conference on Learning@ scale conference, pp 99\u2013108","DOI":"10.1145\/2556325.2566238"},{"key":"10.1007\/s40593-022-00322-1_bib23","doi-asserted-by":"crossref","unstructured":"Kumar, S., Chakrabarti, S., & Roy, S. (2017). Earth mover\u2019s distance pooling over siamese lstms for automatic short answer grading. In: IJCAI, pp 2046\u20132052","DOI":"10.24963\/ijcai.2017\/284"},{"issue":"4","key":"10.1007\/s40593-022-00322-1_bib24","doi-asserted-by":"crossref","first-page":"389","DOI":"10.1023\/A:1025779619903","article-title":"C-rater: Automated scoring of short-answer questions","volume":"37","author":"Leacock","year":"2003","journal-title":"Computers and the Humanities"},{"key":"10.1007\/s40593-022-00322-1_bib25","doi-asserted-by":"crossref","first-page":"13389","DOI":"10.1609\/aaai.v34i09.7062","article-title":"Multiple data augmentation strategies for improving performance on automatic short answer scoring","volume":"34","author":"Lun","year":"2020","journal-title":"Proceedings of the AAAI Conference on Artificial Intelligence"},{"key":"10.1007\/s40593-022-00322-1_bib26","unstructured":"Madnani, N., & Cahill, A. (2018). Automated scoring: Beyond natural language processing. In: Proceedings of the 27th International Conference on Computational Linguistics, pp 1099\u20131109"},{"key":"10.1007\/s40593-022-00322-1_bib27","doi-asserted-by":"crossref","unstructured":"Marvaniya, S., Saha, S., Dhamecha, T.I., Foltz, P., Sindhgatta, R., & Sengupta, B. (2018). Creating scoring rubric from representative student answers for improved short answer grading. In: Proceedings of the 27th ACM International Conference on Information and Knowledge Management, pp 993\u20131002","DOI":"10.1145\/3269206.3271755"},{"key":"10.1007\/s40593-022-00322-1_bib28","unstructured":"Mieskes, M., & Pado, U. (2018). Work smart-reducing effort in short-answer grading. In: Proceedings of the 7th Workshop on NLP for Computer Assisted Language Learning (NLP4CALL 2018) at SLTC, Stockholm, 7th November 2018, Link\u00f6ping University Electronic Press, 152, pp 57\u201368"},{"key":"10.1007\/s40593-022-00322-1_bib29","unstructured":"Mohler, M., Bunescu, R., & Mihalcea, R. (2011). Learning to grade short answer questions using semantic similarity measures and dependency graph alignments. In: Proceedings of the 49th annual meeting of the association for computational linguistics: Human language technologies, pp 752\u2013762"},{"key":"10.1007\/s40593-022-00322-1_bib30","unstructured":"Nielsen, R. D., Ward, W. H., & Martin, J. H. (2008). Learning to assess low-level conceptual understanding. In: Flairs conference, pp 427\u2013432"},{"key":"10.1007\/s40593-022-00322-1_bib31","first-page":"2825","article-title":"Scikit-learn: Machine learning in Python","volume":"12","author":"Pedregosa","year":"2011","journal-title":"Journal of Machine Learning Research"},{"key":"10.1007\/s40593-022-00322-1_bib32","doi-asserted-by":"crossref","unstructured":"Qi, P., Zhang, Y., Zhang, Y., Bolton, J., & Manning, C. D. (2020). Stanza: A Python natural language processing toolkit for many human languages. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics: System Demonstrations. https:\/\/nlp.stanford.edu\/pubs\/qi2020stanza.pdf. Accessed Sep 2020.","DOI":"10.18653\/v1\/2020.acl-demos.14"},{"key":"10.1007\/s40593-022-00322-1_bib33","doi-asserted-by":"crossref","unstructured":"Reimers, N., Gurevych, I. (2019). Sentence-bert: Sentence embeddings using siamese bert-networks. In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing, Association for Computational Linguistics, http:\/\/arxiv.org\/abs\/1908.10084","DOI":"10.18653\/v1\/D19-1410"},{"key":"10.1007\/s40593-022-00322-1_bib34","doi-asserted-by":"crossref","unstructured":"Reimers, N., & Gurevych, I. (2020). Making monolingual sentence embeddings multilingual using knowledge distillation. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing, Association for Computational Linguistics, https:\/\/arxiv.org\/abs\/2004.09813","DOI":"10.18653\/v1\/2020.emnlp-main.365"},{"key":"10.1007\/s40593-022-00322-1_bib35","doi-asserted-by":"crossref","unstructured":"Riordan, B., Horbach, A., Cahill, A., Zesch, T., & Lee, C. (2017). Investigating neural architectures for short answer scoring. In: Proceedings of the 12th Workshop on Innovative Use of NLP for Building Educational Applications, pp 159\u2013168","DOI":"10.18653\/v1\/W17-5017"},{"key":"10.1007\/s40593-022-00322-1_bib36","unstructured":"Settles, B. (2009). Active learning literature survey. Computer Sciences Technical Report 1648, University of Wisconsin\u2013Madison"},{"issue":"4","key":"10.1007\/s40593-022-00322-1_bib37","first-page":"35","article-title":"Modern information retrieval: A brief overview","volume":"24","author":"Singhal","year":"2001","journal-title":"IEEE Data Eng Bull"},{"key":"10.1007\/s40593-022-00322-1_bib38","doi-asserted-by":"crossref","unstructured":"Souza, D. M., Felizardo, K. R., & Barbosa, E. F. (2016). A systematic literature review of assessment tools for programming assignments. In: 2016 IEEE 29Th international conference on software engineering education and training (CSEET), IEEE, pp 147\u2013156","DOI":"10.1109\/CSEET.2016.48"},{"key":"10.1007\/s40593-022-00322-1_bib39","doi-asserted-by":"crossref","unstructured":"Sung, C., Dhamecha, T. I., & Mukhi, N. (2019) Improving short answer grading using transformer-based pre-training. In: International Conference on Artificial Intelligence in Education, Springer, pp 469\u2013481","DOI":"10.1007\/978-3-030-23204-7_39"},{"issue":"3","key":"10.1007\/s40593-022-00322-1_bib40","doi-asserted-by":"crossref","first-page":"5718","DOI":"10.1016\/j.eswa.2008.06.108","article-title":"Cluster-based under-sampling approaches for imbalanced data distributions","volume":"36","author":"Yen","year":"2009","journal-title":"Expert Systems with Applications"},{"key":"10.1007\/s40593-022-00322-1_bib41","doi-asserted-by":"crossref","unstructured":"Zesch, T., Heilman, M., & Cahill, A. (2015). Reducing annotation efforts in supervised short answer scoring. In: Proceedings of the Tenth Workshop on Innovative Use of NLP for Building Educational Applications, pp 124\u2013132","DOI":"10.3115\/v1\/W15-0615"}],"container-title":["International Journal of Artificial Intelligence in Education"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s40593-022-00322-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s40593-022-00322-1\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1560429226002969?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1560429226002969?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s40593-022-00322-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,15]],"date-time":"2026-06-15T17:16:47Z","timestamp":1781543807000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S1560429226002969"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6]]},"references-count":41,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2024,6]]}},"alternative-id":["S1560429226002969"],"URL":"https:\/\/doi.org\/10.1007\/s40593-022-00322-1","relation":{},"ISSN":["1560-4292"],"issn-type":[{"value":"1560-4292","type":"print"}],"subject":[],"published":{"date-parts":[[2024,6]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Reducing Workload in Short Answer Grading Using Machine Learning","name":"articletitle","label":"Article Title"},{"value":"International Journal of Artificial Intelligence in Education","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1007\/s40593-022-00322-1","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"Copyright \u00a9 2023 The Author(s). Published by Elsevier Ltd","name":"copyright","label":"Copyright"}]}}