{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,25]],"date-time":"2026-06-25T04:50:43Z","timestamp":1782363043683,"version":"3.54.5"},"publisher-location":"Cham","reference-count":21,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783031116438","type":"print"},{"value":"9783031116445","type":"electronic"}],"license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022]]},"DOI":"10.1007\/978-3-031-11644-5_38","type":"book-chapter","created":{"date-parts":[[2022,7,26]],"date-time":"2022-07-26T14:18:49Z","timestamp":1658845129000},"page":"465-476","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":14,"title":["Balancing Cost and\u00a0Quality: An Exploration of\u00a0Human-in-the-Loop Frameworks for\u00a0Automated Short Answer Scoring"],"prefix":"10.1007","author":[{"given":"Hiroaki","family":"Funayama","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tasuku","family":"Sato","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yuichiroh","family":"Matsubayashi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tomoya","family":"Mizumoto","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jun","family":"Suzuki","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kentaro","family":"Inui","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2022,7,27]]},"reference":[{"key":"38_CR1","unstructured":"Attali, Y., Burstein, J.: Automated essay scoring with e-rater\u00aev.2. J. Technol. Learn. Assess. 4(3) (2006)"},{"issue":"1","key":"38_CR2","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1007\/s40593-014-0026-8","volume":"25","author":"S Burrows","year":"2015","unstructured":"Burrows, S., Gurevych, I., Stein, B.: The eras and trends of automatic short answer grading. Int. J. Artif. Intell. Educ. 25(1), 60\u2013117 (2015)","journal-title":"Int. J. Artif. Intell. Educ."},{"key":"38_CR3","doi-asserted-by":"publisher","unstructured":"Devlin, J., Chang, M.W., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. In: NAACL-HLT, pp. 4171\u20134186, June 2019. https:\/\/doi.org\/10.18653\/v1\/N19-1423","DOI":"10.18653\/v1\/N19-1423"},{"key":"38_CR4","doi-asserted-by":"publisher","unstructured":"Ding, Y., Riordan, B., Horbach, A., Cahill, A., Zesch, T.: Don\u2019t take \u201cnswvtnvakgxpm\" for an answer -the surprising vulnerability of automatic content scoring systems to adversarial input. In: COLING, pp. 882\u2013892. International Committee on Computational Linguistics, December 2020. https:\/\/doi.org\/10.18653\/v1\/2020.coling-main.76","DOI":"10.18653\/v1\/2020.coling-main.76"},{"key":"38_CR5","doi-asserted-by":"publisher","unstructured":"Funayama, H., et al.: Preventing critical scoring errors in short answer scoring with confidence estimation. In: ACL-SRW, pp. 237\u2013243. Association for Computational Linguistics, July 2020. https:\/\/doi.org\/10.18653\/v1\/2020.acl-srw.32","DOI":"10.18653\/v1\/2020.acl-srw.32"},{"key":"38_CR6","unstructured":"Gardner, J.R., Pleiss, G., Bindel, D., Weinberger, K.Q., Wilson, A.G.: Gpytorch: Blackbox matrix-matrix gaussian process inference with GPU acceleration (2021)"},{"key":"38_CR7","doi-asserted-by":"crossref","unstructured":"Herwanto, G., Sari, Y., Prastowo, B., Riasetiawan, M., Bustoni, I.A., Hidayatulloh, I.: Ukara: a fast and simple automatic short answer scoring system for bahasa indonesia. In: ICEAP Proceeding Book, vol. 2, pp. 48\u201353, December 2018","DOI":"10.26499\/iceap.v2i1.95"},{"key":"38_CR8","doi-asserted-by":"crossref","unstructured":"Horbach, A., Palmer, A.: Investigating active learning for short-answer scoring. In: BEA. Association for Computational Linguistics, San Diego, June 2016","DOI":"10.18653\/v1\/W16-0535"},{"issue":"2","key":"38_CR9","first-page":"226","volume":"2014","author":"ES Jang","year":"2014","unstructured":"Jang, E.S., Kang, S., Noh, E.H., Kim, M.H., Sung, K.H., Seong, T.J.: Kass: korean automatic scoring system for short-answer questions. CSEDU 2014(2), 226\u2013230 (2014)","journal-title":"CSEDU"},{"key":"38_CR10","unstructured":"Jiang, H., Kim, B., Guan, M.Y., Gupta, M.R.: To trust or not to trust a classifier. In: NIPS, pp. 5546\u20135557 (2018)"},{"key":"38_CR11","doi-asserted-by":"publisher","unstructured":"Johan Berggren, S., Rama, T., \u00d8vrelid, L.: Regression or classification? automated essay scoring for Norwegian. In: BEA, pp. 92\u2013102. Association for Computational Linguistics, Florence, August 2019. https:\/\/doi.org\/10.18653\/v1\/W19-4409","DOI":"10.18653\/v1\/W19-4409"},{"key":"38_CR12","first-page":"1712","volume":"7","author":"S Krishnamurthy","year":"2019","unstructured":"Krishnamurthy, S., Gayakwad, E., Kailasanathan, N.: Deep learning for short answer scoring. Int. J. Rec.Technol. Eng. 7, 1712\u20131715 (2019)","journal-title":"Int. J. Rec.Technol. Eng."},{"key":"38_CR13","doi-asserted-by":"publisher","unstructured":"Kumar, Y., Aggarwal, S., Mahata, D., Shah, R.R., Kumaraguru, P., Zimmermann, R.: Get it scored using autosas - an automated system for scoring short answers. In: AAAI\/IAAI\/EAAI. AAAI Press (2019). https:\/\/doi.org\/10.1609\/aaai.v33i01.33019662","DOI":"10.1609\/aaai.v33i01.33019662"},{"key":"38_CR14","doi-asserted-by":"publisher","unstructured":"Leacock, C., Chodorow, M.: C-rater: automated scoring of short-answer questions. Comput. Human. 37(4), 389\u2013405 (2003). https:\/\/doi.org\/10.1023\/A:1025779619903","DOI":"10.1023\/A:1025779619903"},{"key":"38_CR15","doi-asserted-by":"publisher","unstructured":"Mizumoto, T., et al.: Analytic score prediction and justification identification in automated short answer scoring. In: BEA, pp. 316\u2013325 (2019). https:\/\/doi.org\/10.18653\/v1\/W19-4433","DOI":"10.18653\/v1\/W19-4433"},{"key":"38_CR16","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"63","DOI":"10.1007\/978-3-540-28650-9_4","volume-title":"Advanced Lectures on Machine Learning","author":"CE Rasmussen","year":"2004","unstructured":"Rasmussen, C.E.: Gaussian processes in machine learning. In: Bousquet, O., von Luxburg, U., R\u00e4tsch, G. (eds.) ML -2003. LNCS (LNAI), vol. 3176, pp. 63\u201371. Springer, Heidelberg (2004). https:\/\/doi.org\/10.1007\/978-3-540-28650-9_4"},{"key":"38_CR17","doi-asserted-by":"publisher","unstructured":"Riordan, B., Horbach, A., Cahill, A., Zesch, T., Lee, C.M.: Investigating neural architectures for short answer scoring. In: BEA, pp. 159\u2013168 (2017). https:\/\/doi.org\/10.18653\/v1\/W17-5017","DOI":"10.18653\/v1\/W17-5017"},{"key":"38_CR18","doi-asserted-by":"publisher","first-page":"264","DOI":"10.1016\/j.cogsys.2019.09.025","volume":"59","author":"O Sychev","year":"2020","unstructured":"Sychev, O., Anikin, A., Prokudin, A.: Automatic grading and hinting in open-ended text questions. Cogn. Syst. Res. 59, 264\u2013272 (2020)","journal-title":"Cogn. Syst. Res."},{"key":"38_CR19","doi-asserted-by":"publisher","unstructured":"Wang, T., Inoue, N., Ouchi, H., Mizumoto, T., Inui, K.: Inject rubrics into short answer grading system. In: DeepLo, pp. 175\u2013182, November 2019. https:\/\/doi.org\/10.18653\/v1\/D19-6119","DOI":"10.18653\/v1\/D19-6119"},{"issue":"1","key":"38_CR20","doi-asserted-by":"publisher","first-page":"2","DOI":"10.1111\/j.1745-3992.2011.00223.x","volume":"31","author":"DM Williamson","year":"2012","unstructured":"Williamson, D.M., Xi, X., Breyer, F.J.: A framework for evaluation and use of automated scoring. Educ. Meas. Issues Pract. 31(1), 2\u201313 (2012). https:\/\/doi.org\/10.1111\/j.1745-3992.2011.00223.x","journal-title":"Educ. Meas. Issues Pract."},{"key":"38_CR21","doi-asserted-by":"publisher","unstructured":"Woods, B., Adamson, D., Miel, S., Mayfield, E.: Formative essay feedback using predictive scoring models. In: KDD 2017, pp. 2071\u20132080. Association for Computing Machinery (2017). https:\/\/doi.org\/10.1145\/3097983.3098160","DOI":"10.1145\/3097983.3098160"}],"container-title":["Lecture Notes in Computer Science","Artificial Intelligence in Education"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-11644-5_38","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,13]],"date-time":"2024-03-13T19:52:36Z","timestamp":1710359556000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-11644-5_38"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"ISBN":["9783031116438","9783031116445"],"references-count":21,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-11644-5_38","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022]]},"assertion":[{"value":"27 July 2022","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"AIED","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Artificial Intelligence in Education","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Durham","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"United Kingdom","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27 July 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"31 July 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"23","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"aied2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/aied2022.webspace.durham.ac.uk\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"243","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"40","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"40","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"16% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"5","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}