{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,25]],"date-time":"2026-04-25T21:34:35Z","timestamp":1777152875223,"version":"3.51.4"},"publisher-location":"Cham","reference-count":23,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031362712","type":"print"},{"value":"9783031362729","type":"electronic"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-36272-9_34","type":"book-chapter","created":{"date-parts":[[2023,6,25]],"date-time":"2023-06-25T23:03:19Z","timestamp":1687734199000},"page":"414-425","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":6,"title":["Learning When to\u00a0Defer to\u00a0Humans for\u00a0Short Answer Grading"],"prefix":"10.1007","author":[{"given":"Zhaohui","family":"Li","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Chengning","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yumi","family":"Jin","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xuesong","family":"Cang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Sadhana","family":"Puntambekar","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Rebecca J.","family":"Passonneau","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,6,26]]},"reference":[{"key":"34_CR1","unstructured":"Baikadi, A., et al.: An apprenticeship model for human and AI collaborative essay grading. In: Trattner, C., Parra, D., Riche, N. (eds.) Joint Proceedings of the ACM IUI 2019 Workshops Co-located with the 24th ACM Conference on Intelligent User Interfaces (ACM IUI 2019), vol. 2327 (2019). http:\/\/ceur-ws.org\/Vol-2327\/IUI19WS-UIBK-2.pdf"},{"key":"34_CR2","doi-asserted-by":"publisher","unstructured":"Bondi, E., et al.: Role of human-AI interaction in selective prediction. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol. 36 (5), pp. 5286\u20135294 (2022). https:\/\/doi.org\/10.1609\/aaai.v36i5.20465","DOI":"10.1609\/aaai.v36i5.20465"},{"key":"34_CR3","doi-asserted-by":"publisher","unstructured":"Conneau, A., Kiela, D., Schwenk, H., Barrault, L., Bordes, A.: Supervised learning of universal sentence representations from natural language inference data. In: Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 670\u2013680. ACL, Copenhagen (2017). https:\/\/doi.org\/10.18653\/v1\/D17-1070","DOI":"10.18653\/v1\/D17-1070"},{"key":"34_CR4","doi-asserted-by":"publisher","unstructured":"De-Arteaga, M., Fogliato, R., Chouldechova, A.: A Case for humans-in-the-loop: decisions in the presence of erroneous algorithmic scores. In: Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems, pp. 1\u201312. ACM, Honolulu HI USA (2020). https:\/\/doi.org\/10.1145\/3313831.3376638","DOI":"10.1145\/3313831.3376638"},{"key":"34_CR5","doi-asserted-by":"publisher","unstructured":"Devlin, J., Chang, M.W., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. In: Proceedings of the 2019 Conference of the North American Chapter of the ACLm, pp. 4171\u20134186. ACL (2019). https:\/\/doi.org\/10.18653\/v1\/N19-1423","DOI":"10.18653\/v1\/N19-1423"},{"key":"34_CR6","unstructured":"Dzikovska, M., et al.: SemEval-2013 task 7: the joint student response analysis and 8th recognizing textual entailment challenge. In: Proceedings of the Seventh International Workshop on Semantic Evaluation (SemEval 2013), pp. 263\u2013274. ACL (2013). https:\/\/aclanthology.org\/S13-2045"},{"key":"34_CR7","doi-asserted-by":"publisher","unstructured":"Garg, S., Moschitti, A.: Will this question be answered? Question filtering via answer model distillation for efficient question answering. In: Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing, pp. 7329\u20137346. Association for Computational Linguistics (2021). https:\/\/doi.org\/10.18653\/v1\/2021.emnlp-main.583, https:\/\/aclanthology.org\/2021.emnlp-main.583","DOI":"10.18653\/v1\/2021.emnlp-main.583"},{"key":"34_CR8","doi-asserted-by":"publisher","unstructured":"Ghavidel, H., Zouaq, A., Desmarais, M.: Using BERT and XLNET for the automatic short answer grading task. In: Proceedings of the 12th International Conference on Computer Supported Education, pp. 58\u201367. SCITEPRESS - Science and Technology Publications, Prague, Czech Republic (2020). https:\/\/doi.org\/10.5220\/0009422400580067","DOI":"10.5220\/0009422400580067"},{"issue":"2","key":"34_CR9","doi-asserted-by":"publisher","first-page":"179","DOI":"10.3102\/0034654320914744","volume":"90","author":"S Graham","year":"2020","unstructured":"Graham, S., Kiuhara, S.A., MacKay, M.: The effects of writing on learning in science, social studies, and mathematics: a meta-analysis. Rev. Educ. Res. 90(2), 179\u2013226 (2020). https:\/\/doi.org\/10.3102\/0034654320914744","journal-title":"Rev. Educ. Res."},{"key":"34_CR10","doi-asserted-by":"publisher","unstructured":"Green, B., Chen, Y.: Algorithm-in-the-loop decision making. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol. 34(9), pp. 13663\u201313664 (2020). https:\/\/doi.org\/10.1609\/aaai.v34i09.7115","DOI":"10.1609\/aaai.v34i09.7115"},{"key":"34_CR11","unstructured":"Hendrycks, D., Gimpel, K.: A baseline for detecting misclassified and out-of-distribution examples in neural networks. http:\/\/arxiv.org\/abs\/1610.02136, number: arXiv:1610.02136"},{"key":"34_CR12","doi-asserted-by":"publisher","unstructured":"Kamath, A., Jia, R., Liang, P.: Selective question answering under domain shift. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 5684\u20135696. Association for Computational Linguistics (2020). https:\/\/doi.org\/10.18653\/v1\/2020.acl-main.503, https:\/\/aclanthology.org\/2020.acl-main.503","DOI":"10.18653\/v1\/2020.acl-main.503"},{"key":"34_CR13","doi-asserted-by":"publisher","unstructured":"Li, Z., Tomar, Y., Passonneau, R.J.: A Semantic feature-wise transformation relation network for automatic short answer grading. In: Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 6030\u20136040. ACL (2021). https:\/\/doi.org\/10.18653\/v1\/2021.emnlp-main.487","DOI":"10.18653\/v1\/2021.emnlp-main.487"},{"key":"34_CR14","doi-asserted-by":"publisher","first-page":"169","DOI":"10.1007\/978-3-030-23207-8_32","volume-title":"Artificial Intelligence in Education","author":"T Liu","year":"2019","unstructured":"Liu, T., Ding, W., Wang, Z., Tang, J., Huang, G.Y., Liu, Z.: Automatic short answer grading via multiway attention networks. In: Isotani, S., Mill\u00e1n, E., Ogan, A., Hastings, P., McLaren, B., Luckin, R. (eds.) Artificial Intelligence in Education, pp. 169\u2013173. Springer International Publishing, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-23207-8_32"},{"key":"34_CR15","doi-asserted-by":"publisher","DOI":"10.1007\/s10462-022-10246-w","author":"E Mosqueira-Rey","year":"2022","unstructured":"Mosqueira-Rey, E., Hern\u00e1ndez-Pereira, E., Alonso-R\u00edos, D., Bobes-Bascar\u00e1n, J., Fern\u00e1ndez-Leal, A.: Human-in-the-loop machine learning: a state of the art. Artif. Intell. Rev. (2022). https:\/\/doi.org\/10.1007\/s10462-022-10246-w","journal-title":"Artif. Intell. Rev."},{"key":"34_CR16","doi-asserted-by":"publisher","first-page":"311","DOI":"10.1162\/tacl_a_00185","volume":"2","author":"RJ Passonneau","year":"2014","unstructured":"Passonneau, R.J., Carpenter, B.: The benefits of a model of annotation. Trans. ACL 2, 311\u2013326 (2014). https:\/\/doi.org\/10.1162\/tacl_a_00185","journal-title":"Trans. ACL"},{"key":"34_CR17","doi-asserted-by":"publisher","unstructured":"Riordan, B., Horbach, A., Cahill, A., Zesch, T., Lee, C.M.: Investigating neural architectures for short answer scoring. In: Proceedings of the 12th Workshop on Innovative Use of NLP for Building Educational Applications, pp. 159\u2013168. ACL, Copenhagen (2017). https:\/\/doi.org\/10.18653\/v1\/W17-5017","DOI":"10.18653\/v1\/W17-5017"},{"key":"34_CR18","doi-asserted-by":"publisher","first-page":"503","DOI":"10.1007\/978-3-319-93843-1_37","volume-title":"Artificial Intelligence in Education","author":"S Saha","year":"2018","unstructured":"Saha, S., Dhamecha, T.I., Marvaniya, S., Sindhgatta, R., Sengupta, B.: Sentence level or token level features for automatic short answer grading?: Use both. In: Penstein Ros\u00e9, C., et al. (eds.) Artificial Intelligence in Education, pp. 503\u2013517. Springer International Publishing, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-319-93843-1_37"},{"key":"34_CR19","doi-asserted-by":"publisher","unstructured":"Sung, C., Dhamecha, T., Saha, S., Ma, T., Reddy, V., Arora, R.: Pre-training BERT on domain resources for short answer grading. In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), pp. 6071\u20136075. ACL, Hong Kong (2019). https:\/\/doi.org\/10.18653\/v1\/D19-1628","DOI":"10.18653\/v1\/D19-1628"},{"key":"34_CR20","doi-asserted-by":"publisher","unstructured":"Varshney, N., Mishra, S., Baral, C.: Investigating selective prediction approaches across several tasks in IID, OOD, and adversarial settings. In: Findings of the Association for Computational Linguistics: ACL 2022, pp. 1995\u20132002. Association for Computational Linguistics, Dublin, Ireland (2022). https:\/\/doi.org\/10.18653\/v1\/2022.findings-acl.158","DOI":"10.18653\/v1\/2022.findings-acl.158"},{"key":"34_CR21","doi-asserted-by":"publisher","unstructured":"Wang, T., Inoue, N., Ouchi, H., Mizumoto, T., Inui, K.: Inject rubrics into short answer grading system. In: Proceedings of the 2nd Workshop on Deep Learning Approaches for Low-Resource NLP (DeepLo 2019), pp. 175\u2013182. ACL, Hong Kong (2019). https:\/\/doi.org\/10.18653\/v1\/D19-6119","DOI":"10.18653\/v1\/D19-6119"},{"key":"34_CR22","unstructured":"Wiener, Y.: Theoretical foundations of selective prediction. Ph.D. thesis, Technion - Israel Institute of Technology, Israel (2013)"},{"key":"34_CR23","doi-asserted-by":"publisher","unstructured":"Xin, J., Tang, R., Yu, Y., Lin, J.: The art of abstention: selective prediction and error regularization for natural language processing. In: Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics (ACL) and the 11th International Joint Conference on Natural Language Processing (IJCNLP), pp. 1040\u20131051. Association for Computational Linguistics (2021). https:\/\/doi.org\/10.18653\/v1\/2021.acl-long.84","DOI":"10.18653\/v1\/2021.acl-long.84"}],"container-title":["Lecture Notes in Computer Science","Artificial Intelligence in Education"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-36272-9_34","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,12]],"date-time":"2024-03-12T16:08:47Z","timestamp":1710259727000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-36272-9_34"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031362712","9783031362729"],"references-count":23,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-36272-9_34","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"26 June 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"AIED","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Artificial Intelligence in Education","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Tokyo","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Japan","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"3 July 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"7 July 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"24","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"aied2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.aied2023.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Easychair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"311","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"53","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"26","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"17% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"5","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}