{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,17]],"date-time":"2026-03-17T22:42:00Z","timestamp":1773787320604,"version":"3.50.1"},"reference-count":45,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2024,6,4]],"date-time":"2024-06-04T00:00:00Z","timestamp":1717459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,6,4]],"date-time":"2024-06-04T00:00:00Z","timestamp":1717459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Int J Data Sci Anal"],"published-print":{"date-parts":[[2025,9]]},"DOI":"10.1007\/s41060-024-00576-z","type":"journal-article","created":{"date-parts":[[2024,6,3]],"date-time":"2024-06-03T23:54:53Z","timestamp":1717458893000},"page":"1871-1883","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["SPRAG: building and benchmarking a Short Programming-Related Answer Grading dataset"],"prefix":"10.1007","volume":"20","author":[{"given":"Sridevi","family":"Bonthu","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"S. Rama","family":"Sree","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"M. H. M. Krishna","family":"Prasad","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,6,4]]},"reference":[{"issue":"11","key":"576_CR1","doi-asserted-by":"publisher","first-page":"5817","DOI":"10.3390\/su13115817","volume":"13","author":"CM Stracke","year":"2021","unstructured":"Stracke, C.M., Trisolini, G.: A systematic literature review on the quality of MOOCs. Sustainability 13(11), 5817 (2021)","journal-title":"Sustainability"},{"key":"576_CR2","doi-asserted-by":"publisher","first-page":"61","DOI":"10.1007\/978-3-030-84060-0_5","volume-title":"Machine learning and knowledge extraction","author":"S Bonthu","year":"2021","unstructured":"Bonthu, S., Rama Sree, S., Prasad, M.K.: Automated short answer grading using deep learning: a survey. In: Holzinger, A., Peter Kieseberg, A., Tjoa, M., Weippl, E. (eds.) Machine learning and knowledge extraction, pp. 61\u201378. Springer, Cham (2021)"},{"key":"576_CR3","doi-asserted-by":"publisher","first-page":"1649959","DOI":"10.1080\/10872981.2019.1649959","volume":"24","author":"JB Bird","year":"2019","unstructured":"Bird, J.B., Olvet, D.M., Willey, J.M., Brenner, J.: Patients don\u2019t come with multiple choice options: essay-based assessment in UME. Med. Educ. Online 24, 1649959 (2019)","journal-title":"Med. Educ. Online"},{"issue":"1","key":"576_CR4","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1007\/s40593-014-0026-8","volume":"25","author":"S Burrows","year":"2015","unstructured":"Burrows, S., Gurevych, I., Stein, B.: The eras and trends of automatic short answer grading. Int. J. Artif. Intell. Educ.Artif. Intell. Educ. 25(1), 60\u2013117 (2015). https:\/\/doi.org\/10.1007\/s40593-014-0026-8","journal-title":"Int. J. Artif. Intell. Educ.Artif. Intell. Educ."},{"issue":"2","key":"576_CR5","doi-asserted-by":"publisher","first-page":"31","DOI":"10.3991\/ijet.v16i02.18739","volume":"16","author":"W Wu","year":"2021","unstructured":"Wu, W., Berestova, A., Lobuteva, A., Stroiteleva, N.: An intelligent computer system for assessing student performance. Int. J. Emerg Technol. Learn. (iJET) 16(2), 31\u201345 (2021)","journal-title":"Int. J. Emerg Technol. Learn. (iJET)"},{"key":"576_CR6","doi-asserted-by":"publisher","first-page":"67","DOI":"10.1007\/s10579-015-9313-8","volume":"50","author":"MO Dzikovska","year":"2016","unstructured":"Dzikovska, M.O., Nielsen, R.D., Leacock, C.: The joint student response analysis and recognizing textual entailment challenge: making sense of student responses in educational applications. Lang. Resour. Eval.Resour. Eval. 50, 67\u201393 (2016)","journal-title":"Lang. Resour. Eval.Resour. Eval."},{"key":"576_CR7","doi-asserted-by":"crossref","unstructured":"Mohler, M., Mihalcea, R.: Text-to-text semantic similarity for automatic short answer grading. In: Proceedings of the 12th Conference of the European Chapter of the ACL (EACL 2009), pp. 567\u2013575. (2009)","DOI":"10.3115\/1609067.1609130"},{"key":"576_CR8","unstructured":"Mohler, M., Bunescu, R., Mihalcea, R.: Learning to grade short answer questions using semantic similarity measures and dependency graph alignments. In: Proceedings of the 49th annual meeting of the association for computational linguistics: Human language technologies, pp. 752\u2013762. (2011)"},{"key":"576_CR9","doi-asserted-by":"publisher","first-page":"391","DOI":"10.1162\/tacl_a_00236","volume":"1","author":"S Basu","year":"2013","unstructured":"Basu, S., Jacobs, C., Vanderwende, L.: Powergrading: a clustering approach to amplify human effort for short answer grading. Trans. Assoc. Comput. Linguist. 1, 391\u2013402 (2013)","journal-title":"Trans. Assoc. Comput. Linguist."},{"key":"576_CR10","unstructured":"Ouahrani, L a. B. D.: AR-ASAG an Arabic dataset for automatic short answer grading evaluation. In: Proceedings of the Twelfth Language Resources and Evaluation Conference, pp. 2634\u20132643. (2020)"},{"key":"576_CR11","doi-asserted-by":"crossref","unstructured":"Gomaa, W. H., Fahmy, A. A.: Arabic short answer scoring with effective feedback for students. In: International Journal of Computer Applications, vol. 86, pp. 35\u201341 Foundation of Computer Science. (2014)","DOI":"10.5120\/14961-3177"},{"key":"576_CR12","unstructured":"Sarkar, M. O. T., Thompson, C., Rayner, G.: Graduate employability: views of recent science graduates and employers. Int. J. Innov. Sci. Math. Educ., 24(3), 2016"},{"key":"576_CR13","doi-asserted-by":"crossref","unstructured":"Reimers, N., Gurevych, I.: Sentence-bert: Sentence embeddings using siamese bert-networks. arXiv preprint arXiv:1908.10084, (2019)","DOI":"10.18653\/v1\/D19-1410"},{"key":"576_CR14","unstructured":"Haller, S., Aldea, A., Seifert, C., Strisciuglio, N.: Survey on automated short answer grading with deep learning: from word embeddings to transformers. Preprint at https:\/\/arxiv.org\/abs\/2204.03503, (2022)"},{"key":"576_CR15","unstructured":"Dzikovska, M. O., Nielsen, R., Brew, C., Leacock, C., Giampiccolo, D., Bentivogli, L., Clark, P., Dagan, I., Dang, H. T.: Semeval-2013 task 7: The joint student response analysis and 8th recognizing textual entailment challenge, In: Second Joint Conference on Lexical and Computational Semantics (* SEM), Proceedings of the Seventh International Workshop on Semantic Evaluation, vol. 2, pp. 263\u2013274. SemEval (2013)"},{"issue":"1","key":"576_CR16","doi-asserted-by":"publisher","first-page":"88","DOI":"10.1007\/s40593-022-00289-z","volume":"33","author":"J Schneider","year":"2023","unstructured":"Schneider, J., Richner, R., Riser, M.: Towards trustworthy autograding of short, multi-lingual, multi-type answers. Int. J. Artif. Intell. Educ.Artif. Intell. Educ. 33(1), 88\u2013118 (2023)","journal-title":"Int. J. Artif. Intell. Educ.Artif. Intell. Educ."},{"key":"576_CR17","unstructured":"Agarwal, D., Gupta, S., Baghel, N.: ScAA: a dataset for automated short answer grading of children\u2019s free-text answers in Hindi and Marathi. In: Proceedings of the 17th International Conference on Natural Language Processing (ICON), pp. 430\u2013436. (2020)"},{"key":"576_CR18","unstructured":"Baral, S., Botelho, A. F., Erickson, J. A., Benachamardi, P., Heffernan, N. T.: Improving automated scoring of student open responses in mathematics. International Educational Data Mining Society, (2021)"},{"key":"576_CR19","doi-asserted-by":"crossref","unstructured":"Galhardi, L., de Souza, R. C. T., Brancher, J.: Automatic grading of Portuguese short answers using a machine learning approach. In: Anais Estendidos do XVI Simp{\\'o}sio Brasileiro de Sistemas de Informa{\\c{c}}{\\~a}o, SBC, pp. 109\u2013124. (2020)","DOI":"10.5753\/sbsi.2020.13133"},{"key":"576_CR20","doi-asserted-by":"crossref","unstructured":"Kumar, S., Chakrabarti, S., Roy, S.: Earth mover's distance pooling over siamese LSTMs for automatic short answer grading. In: IJCAI, pp. 2046\u20132052. (2017)","DOI":"10.24963\/ijcai.2017\/284"},{"key":"576_CR21","doi-asserted-by":"crossref","unstructured":"Zhao, S., Zhang, Y., Xiong, X., Botelho, A., Heffernan, N.: A memory-augmented neural model for automated grading. In: Proceedings of the fourth (2017) ACM conference on learning@ scale, pp. 189\u2013192. (2017)","DOI":"10.1145\/3051457.3053982"},{"key":"576_CR22","doi-asserted-by":"crossref","unstructured":"Riordan, B., Horbach, A., Cahill, A., Zesch, T., Lee, C.: Investigating neural architectures for short answer scoring. In: Proceedings of the 12th workshop on innovative use of NLP for building educational applications, pp. 159\u2013168. (2017)","DOI":"10.18653\/v1\/W17-5017"},{"key":"576_CR23","doi-asserted-by":"crossref","unstructured":"Tan, C., Wei, F., Wang, W., Lv, W., Zhou, M.: Multiway attention networks for modeling sentence pairs. In: IJCAI, pp. 4411\u20134417. (2018)","DOI":"10.24963\/ijcai.2018\/613"},{"key":"576_CR24","doi-asserted-by":"crossref","unstructured":"Sung, C., Dhamecha, T. I., Mukhi, N.: Improving short answer grading using transformer-based pre-training. In: Artificial Intelligence in Education: 20th International Conference, Proceedings, Part I 20, AIED 2019, pp. 469\u2013481, Chicago, IL, 25\u201329 June, Springer (2019)","DOI":"10.1007\/978-3-030-23204-7_39"},{"key":"576_CR25","unstructured":"Condor, A., Litster, M., Pardos, Z., Automatic Short Answer Grading with SBERT on Out-of-Sample Questions. International Educational Data Mining Society, (2021)"},{"key":"576_CR26","unstructured":"Hoek, J., Scholman, M. Evaluating discourse annotation: Some recent insights and new approaches. In: Proceedings of the 13th Joint ISO-ACL Workshop on Interoperable Semantic Annotation (isa-13), (2017)"},{"key":"576_CR27","doi-asserted-by":"publisher","first-page":"2","DOI":"10.5334\/dsj-2015-002","volume":"14","author":"L Cai","year":"2015","unstructured":"Cai, L., Zhu, Y.: The challenges of data quality and data quality assessment in the big data era. Data Sci. J. 14, 2\u20132 (2015)","journal-title":"Data Sci. J."},{"key":"576_CR28","first-page":"1720","volume":"2022","author":"A Braylan","year":"2022","unstructured":"Braylan, A., Alonso, O., Lease, M.: Measuring annotator agreement generally across complex structured, multi-object, and free-text annotation tasks. Proc. ACM Web Conf. 2022, 1720\u20131730 (2022)","journal-title":"Proc. ACM Web Conf."},{"issue":"6","key":"576_CR29","doi-asserted-by":"publisher","first-page":"2611","DOI":"10.1177\/0962280214529560","volume":"25","author":"D Marasini","year":"2016","unstructured":"Marasini, D., Quatto, P., Ripamonti, E.: Assessing the inter-rater agreement for ordinal data through weighted indexes. Stat. Methods Med. Res. 25(6), 2611\u20132633 (2016)","journal-title":"Stat. Methods Med. Res."},{"issue":"5","key":"576_CR30","first-page":"360","volume":"37","author":"AJ Viera","year":"2005","unstructured":"Viera, A.J., Garrett, J.M., et al.: Understanding interobserver agreement: the kappa statistic. Fam med 37(5), 360\u2013363 (2005)","journal-title":"Fam med"},{"key":"576_CR31","volume-title":"Statistical methods for rates and proportions","author":"JL Fleiss","year":"2013","unstructured":"Fleiss, J.L., Levin, B., Paik, M.C.: Statistical methods for rates and proportions. Wiley, New York (2013)"},{"key":"576_CR32","doi-asserted-by":"publisher","first-page":"79","DOI":"10.1007\/s40593-017-0142-3","volume":"28","author":"V Sowmya","year":"2018","unstructured":"Sowmya, V.: Automated assessment of non-native learner essays: investigating the role of linguistic features. Int. J. Artif. Intell. Educ.Artif. Intell. Educ. 28, 79\u2013105 (2018)","journal-title":"Int. J. Artif. Intell. Educ.Artif. Intell. Educ."},{"issue":"1","key":"576_CR33","doi-asserted-by":"publisher","first-page":"77","DOI":"10.1109\/TLT.2019.2897997","volume":"13","author":"A Sahu","year":"2019","unstructured":"Sahu, A., Bhowmick, P.K.: Feature engineering and ensemble-based approach for improving automatic short-answer grading performance. IEEE Trans. Learn. Technol. 13(1), 77\u201390 (2019)","journal-title":"IEEE Trans. Learn. Technol."},{"key":"576_CR34","unstructured":"Devlin, J., Chang, M.-W., Lee, K., Toutanova, K.: Bert: pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805 (2018)"},{"key":"576_CR35","unstructured":"Liu, Y., Ott, M., Goyal, N., Du, J., Joshi, M., Chen, D., Levy, O., Lewis, M., Zettlemoyer, L., Stoyanov, V.: Roberta: a robustly optimized bert pretraining approach. arXiv preprint arXiv:1907.11692 (2019)"},{"key":"576_CR36","unstructured":"Sanh, V., Debut, L., Chaumond, J., Wolf, T.: DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter. arXiv preprint arXiv:1910.01108 (2019)"},{"key":"576_CR37","doi-asserted-by":"crossref","unstructured":"Conneau, A., Khandelwal, K., Goyal, N., Chaudhary, V., Wenzek, G., Guzm'an, F., Grave, E., Ott, M., Zettlemoyer, L., Stoyanov, V.: Unsupervised cross-lingual representation learning at scale. arXiv preprint arXiv:1911.02116, (2019)","DOI":"10.18653\/v1\/2020.acl-main.747"},{"key":"576_CR38","unstructured":"Lundberg, S. M., Lee, S.-I.: A unified approach to interpreting model predictions. Advances in neural information processing systems, 30, (2017)"},{"key":"576_CR39","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1007\/s13369-022-07266-7","volume":"48","author":"AO Mohammad","year":"2023","unstructured":"Mohammad, A.O., Azzeh, M.: Toward fluent Arabic poem generation based on fine-tuning AraGPT2 transformer. Arab. J. Sci. Eng. 48, 1\u201313 (2023)","journal-title":"Arab. J. Sci. Eng."},{"issue":"2","key":"576_CR40","doi-asserted-by":"publisher","first-page":"869","DOI":"10.3390\/app11020869","volume":"11","author":"S Shaikh","year":"2021","unstructured":"Shaikh, S., Daudpota, S.M., Imran, A.S., Kastrati, Z.: Towards improved classification accuracy on highly imbalanced text dataset using deep neural language models. Appl. Sci. 11(2), 869 (2021)","journal-title":"Appl. Sci."},{"issue":"7","key":"576_CR41","doi-asserted-by":"publisher","first-page":"2940","DOI":"10.1109\/TNNLS.2020.3047335","volume":"33","author":"KRM Fernando","year":"2021","unstructured":"Fernando, K.R.M., Tsokos, C.P.: Dynamically weighted balanced loss: class imbalanced learning and confidence calibration of deep neural networks. IEEE Trans. Neural Netw. Learn. Syst. 33(7), 2940\u20132951 (2021)","journal-title":"IEEE Trans. Neural Netw. Learn. Syst."},{"key":"576_CR42","doi-asserted-by":"publisher","first-page":"34236","DOI":"10.1109\/ACCESS.2022.3161510","volume":"10","author":"M Park","year":"2022","unstructured":"Park, M., Song, H.J., Kang, D.-O.: imbalanced classification via feature dictionary-based minority oversampling. IEEE Access 10, 34236\u201334245 (2022)","journal-title":"IEEE Access"},{"key":"576_CR43","unstructured":"Cao, C., Zhou, F., Dai, Y., Wang, J.: A survey of mix-based data augmentation: Taxonomy, methods, applications, and explainability. arXiv preprint arXiv:2212.10888 (2022)"},{"issue":"1","key":"576_CR44","doi-asserted-by":"publisher","first-page":"99","DOI":"10.1007\/s13278-022-00934-4","volume":"12","author":"T Ahmed","year":"2022","unstructured":"Ahmed, T., Ivan, S., Kabir, M., Mahmud, H., Hasan, K.: Performance analysis of transformer-based architectures and their ensembles to detect trait-based cyberbullying. Soc. Netw. Anal. Min.Netw. Anal. Min. 12(1), 99 (2022)","journal-title":"Soc. Netw. Anal. Min.Netw. Anal. Min."},{"issue":"6","key":"576_CR45","doi-asserted-by":"publisher","first-page":"2623","DOI":"10.1021\/acs.jcim.1c00160","volume":"61","author":"C Esposito","year":"2021","unstructured":"Esposito, C., Landrum, G.A., Schneider, N., Stiefl, N., Riniker, S.: GHOST: adjusting the decision threshold to handle imbalanced data in machine learning. J. Chem. Inf. Model. 61(6), 2623\u20132640 (2021)","journal-title":"J. Chem. Inf. Model."}],"container-title":["International Journal of Data Science and Analytics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s41060-024-00576-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s41060-024-00576-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s41060-024-00576-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,5]],"date-time":"2025-09-05T19:58:40Z","timestamp":1757102320000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s41060-024-00576-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,4]]},"references-count":45,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2025,9]]}},"alternative-id":["576"],"URL":"https:\/\/doi.org\/10.1007\/s41060-024-00576-z","relation":{},"ISSN":["2364-415X","2364-4168"],"issn-type":[{"value":"2364-415X","type":"print"},{"value":"2364-4168","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,6,4]]},"assertion":[{"value":"24 February 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 May 2024","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"4 June 2024","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The author declares that there are no competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"The research involved student participants and two human graders. Prior to participation in this study, all participants were provided with a detailed explanation of the research objectives, procedures, potential risks, and benefits.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent to humans"}},{"value":"This research did not involve animal subjects.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent to animals"}}]}}