{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,20]],"date-time":"2026-02-20T18:31:55Z","timestamp":1771612315967,"version":"3.50.1"},"reference-count":53,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2015,8,13]],"date-time":"2015-08-13T00:00:00Z","timestamp":1439424000000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2015,8,13]],"date-time":"2015-08-13T00:00:00Z","timestamp":1439424000000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/100000006","name":"Office of Naval Research","doi-asserted-by":"publisher","award":["N000141410733"],"award-info":[{"award-number":["N000141410733"]}],"id":[{"id":"10.13039\/100000006","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000138","name":"U.S. Department of Education","doi-asserted-by":"publisher","award":["R305A120808"],"award-info":[{"award-number":["R305A120808"]}],"id":[{"id":"10.13039\/100000138","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Lang Resources & Evaluation"],"published-print":{"date-parts":[[2016,3]]},"DOI":"10.1007\/s10579-015-9313-8","type":"journal-article","created":{"date-parts":[[2015,8,12]],"date-time":"2015-08-12T04:10:33Z","timestamp":1439352633000},"page":"67-93","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":17,"title":["The joint student response analysis and recognizing textual entailment challenge: making sense of student responses in educational applications"],"prefix":"10.1007","volume":"50","author":[{"given":"Myroslava O.","family":"Dzikovska","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Rodney D.","family":"Nielsen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Claudia","family":"Leacock","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2015,8,13]]},"reference":[{"key":"9313_CR1","unstructured":"Agirre, E., Cer, D., Diab, M., & Gonzalez-Agirre, A. (2012). Semeval-2012 task 6: A pilot on semantic textual similarity. In *SEM 2012: The first joint conference on lexical and computational semantics (pp. 385\u2013393). Montr\u00e9al: Association for Computational Linguistics."},{"key":"9313_CR2","unstructured":"Agirre, E., Cer, D., Diab, M., Gonzalez-Agirre, A., & Guo, W. (2013). *SEM 2013 shared task: Semantic textual similarity. In Second joint conference on lexical and computational semantics (*SEM) (pp. 32\u201343). Atlanta, GA: Association for Computational Linguistics."},{"key":"9313_CR3","unstructured":"Aldabe, I., Maritxalar, M., & Lopez de Lacalle, O. (2013). EHU-ALM: Similarity-feature based approach for student response analysis. In Second joint conference on lexical and computational semantics (*SEM), Vol. 2: Proceedings of the seventh international workshop on semantic evaluation (SemEval 2013) (pp. 580\u2013584). Atlanta, GA: Association for Computational Linguistics."},{"key":"9313_CR5","unstructured":"Bentivogli, L., Clark, P., Dagan, I., Dang, H. T., & Giampiccolo, D. (2010). The sixth PASCAL recognizing textual entailment challenge. In Notebook papers and results, text analysis conference (TAC)."},{"key":"9313_CR6","unstructured":"Bentivogli, L., Clark, P., Dagan, I., Dang, H. T., & Giampiccolo, D. (2011). The seventh PASCAL recognizing textual entailment challenge. In Notebook papers and results, text analysis conference (TAC)."},{"key":"9313_CR4","unstructured":"Bentivogli, L., Dagan, I., Dang, H. T., Giampiccolo, D., & Magnini, B. (2009). The fifth PASCAL recognizing textual entailment challenge. In Proceedings of text analysis conference (TAC) 2009."},{"key":"9313_CR7","unstructured":"Bicici, E., & van Genabith, J. (2013). CNGL: Grading student answers by acts of translation. In Second joint conference on lexical and computational semantics (*SEM), Vol. 2: Proceedings of the seventh international workshop on semantic evaluation (SemEval 2013) (pp. 585\u2013591). Atlanta, GA: Association for Computational Linguistics."},{"issue":"1","key":"9313_CR8","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1007\/s40593-014-0026-8","volume":"25","author":"S Burrows","year":"2015","unstructured":"Burrows, S., Gurevych, I., & Stein, B. (2015a). The eras and trends of automatic short answer grading. International Journal of Artificial Intelligence in Education, 25(1), 60\u2013117. doi: 10.1007\/s40593-014-0026-8 .","journal-title":"International Journal of Artificial Intelligence in Education"},{"key":"9313_CR10","volume-title":"Handbook of automated essay evaluation: Current applications and new directions","author":"J Burstein","year":"2013","unstructured":"Burstein, J., Tetreault, J., & Madnani, N. (2013). The e-rater essay scoring system. In M. D. Shermis & J. Burstein (Eds.), Handbook of automated essay evaluation: Current applications and new directions. London: Taylor and Francis."},{"key":"9313_CR11","unstructured":"Campbell, G. C., Steinhauser, N. B., Dzikovska, M. O., Moore, J. D., Callaway, C. B., & Farrow, E. (2009). The DeMAND coding scheme: A \u201ccommon language\u201d for representing and analyzing student discourse. In Proceedings of 14th international conference on artificial intelligence in education (AIED), poster session, Brighton."},{"key":"9313_CR12","doi-asserted-by":"publisher","first-page":"27:1","DOI":"10.1145\/1961189.1961199","volume":"2","author":"CC Chang","year":"2011","unstructured":"Chang, C. C., & Lin, C. J. (2011). LIBSVM: A library for support vector machines. ACM Transactions on Intelligent Systems and Technology, 2, 27:1\u201327:27.","journal-title":"ACM Transactions on Intelligent Systems and Technology"},{"key":"9313_CR13","unstructured":"Dagan, I., Glickman, O., & Magnini, B. (2006). The PASCAL recognising textual entailment challenge. In J. Quin\u0304onero-Candela, I. Dagan, B. Magnini, & F. d\u2019Alch\u00e9 Buc (eds.) Machine learning challenges, lecture notes in computer science (Vol. 3944). Berlin: Springer."},{"key":"9313_CR15","unstructured":"Dale, R., Anisimoff, I., & Narroway, G. (2012). HOO 2012: A report on the preposition and determiner error correction shared task. In Proceedings of the seventh workshop of building educational applications using NLP. Association for Computational Linguistics."},{"key":"9313_CR14","unstructured":"Dale, R., & Kilgarriff, A. (2011). Helping our own: The HOO 2011 pilot shared task. In Proceedings of the generation challenges session at the 13th European workshop on natural language generation (pp. 242\u2013249). Association for Computational Linguistics."},{"key":"9313_CR16","unstructured":"Daume, H., III (2007). Frustratingly easy domain adaptation. In Proceedings of the 45th annual meeting of the Association of Computational Linguistics (pp. 256\u2013263). Association for Computational Linguistics, Prague."},{"key":"9313_CR19","unstructured":"Dzikovska, M. O., Bell, P., Isard, A., Moore, J. D. (2012a). Evaluating language understanding accuracy with respect to objective outcomes in a dialogue system. In Proceedings of EACL-12 conference (pp. 471\u2013481)."},{"key":"9313_CR21","doi-asserted-by":"crossref","unstructured":"Dzikovska, M. O., Farrow, E., & Moore. J. D. (2013a). Combining semantic interpretation and statistical classification for improved explanation processing in a tutorial dialogue system. In Proceedings of the 16th international conference on artificial intelligence in education (AIED 2013), Memphis, TN.","DOI":"10.1007\/978-3-642-39112-5_29"},{"key":"9313_CR18","unstructured":"Dzikovska, M. O., Moore, J. D., Steinhauser, N., Campbell, G., Farrow, E., & Callaway, C. B. (2010). Beetle II: A system for tutoring and computational linguistics experimentation. In Proceedings of ACL 2010 system demonstrations (pp. 13\u201318)."},{"key":"9313_CR20","unstructured":"Dzikovska, M. O., Nielsen, R. D., & Brew, C. (2012b). Towards effective tutorial feedback for explanation questions: A dataset and baselines. In Proceedings of 2012 conference of NAACL: Human language technologies (pp. 200\u2013210)."},{"key":"9313_CR22","unstructured":"Dzikovska, M. O., Nielsen, R., Brew, C., Leacock, C., Giampiccolo, D., Bentivogli, L., et al. (2013b). Semeval-2013 task 7: The joint student response analysis and 8th recognizing textual entailment challenge. In Proceedings of the 6th international workshop on semantic evaluation (SEMEVAL-2013). Association for Computational Linguistics, Atlanta, GA."},{"issue":"3","key":"9313_CR17","doi-asserted-by":"publisher","first-page":"284","DOI":"10.1007\/s40593-014-0017-9","volume":"24","author":"M Dzikovska","year":"2014","unstructured":"Dzikovska, M., Steinhauser, N., Farrow, E., Moore, J., & Campbell, G. (2014). BEETLE II: Deep natural language understanding and automatic feedback generation for intelligent tutoring in basic electricity and electronics. International Journal of Artificial Intelligence in Education, 24(3), 284\u2013332. doi: 10.1007\/s40593-014-0026-8 .","journal-title":"International Journal of Artificial Intelligence in Education"},{"key":"9313_CR23","unstructured":"Giampiccolo, D., Dang, H. T., Magnini, B., Dagan, I., Cabrio, E., & Dolan , B. (2008). The fourth PASCAL recognizing textual entailment challenge. In Proceedings of text analysis conference (TAC) 2008, Gaithersburg, MD."},{"key":"9313_CR24","unstructured":"Glass, M. (2000). Processing language input in the CIRCSIM-Tutor intelligent tutoring system. In Papers from the 2000 AAAI fall symposium. AAAI technical report FS-00-01 (pp. 74\u201379)."},{"key":"9313_CR25","unstructured":"Gleize, M., & Grau, B. (2013). LIMSIILES: Basic English substitution for student answer assessment at semeval 2013. In Second joint conference on lexical and computational semantics (*SEM), Vol. 2: Proceedings of the seventh international workshop on semantic evaluation (SemEval 2013) (pp. 598\u2013602). Atlanta, GA: Association for Computational Linguistics."},{"key":"9313_CR26","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1016\/S1389-0417(99)00005-4","volume":"1","author":"AC Graesser","year":"1999","unstructured":"Graesser, A. C., Wiemer-Hastings, K., Wiemer-Hastings, P., & Kreuz, R. (1999). Autotutor: A simulation of a human tutor. Cognitive Systems Research, 1, 35\u201351.","journal-title":"Cognitive Systems Research"},{"key":"9313_CR27","unstructured":"Heilman, M., & Madnani, N. (2012). ETS: Discriminative edit models for paraphrase scoring. In *SEM 2012: The first joint conference on lexical and computational semantics\u2014Vol. 1: Proceedings of the main conference and the shared task, and Vol. 2: Proceedings of the sixth international workshop on semantic evaluation (SemEval 2012) (pp. 529\u2013535). Montr\u00e9al: Association for Computational Linguistics."},{"key":"9313_CR28","unstructured":"Heilman, M., & Madnani, N. (2013a). ETS: Domain adaptation and stacking for short answer scoring. In Second joint conference on lexical and computational semantics (*SEM), Vol. 2: Proceedings of the seventh international workshop on semantic evaluation (SemEval 2013) (pp. 275\u2013279). Atlanta, GA: Association for Computational Linguistics."},{"key":"9313_CR29","unstructured":"Heilman, M., & Madnani, N. (2013b). HENRY-CORE: Domain adaptation and stacking for text similarity. In Second joint conference on lexical and computational semantics (*SEM), Vol. 1: Proceedings of the main conference and the shared task: semantic textual similarity (pp. 96\u2013102). Atlanta, GA: Association for Computational Linguistics."},{"key":"9313_CR30","unstructured":"Jimenez, S., Becerra, C., & Gelbukh, A. (2013). SOFTCARDINALITY: Hierarchical text overlap for student response analysis. In Second joint conference on lexical and computational semantics (*SEM), Vol. 2: Proceedings of the seventh international workshop on semantic evaluation (SemEval 2013) (pp. 280\u2013284). Atlanta, GA: Association for Computational Linguistics."},{"key":"9313_CR32","doi-asserted-by":"crossref","unstructured":"Jordan, P. W., Makatchev, M., & Pappuswamy, U. (2006a). Understanding complex natural language explanations in tutorial applications. In Proceedings of the third workshop on scalable natural language understanding, ScaNaLU \u201906 (pp. 17\u201324).","DOI":"10.3115\/1621459.1621463"},{"key":"9313_CR31","unstructured":"Jordan, P., Makatchev, M., Pappuswamy, U., VanLehn, K., & Albacete, P. (2006b). A natural language tutorial dialogue system for physics. In Proceedings of the 19th international FLAIRS conference (pp. 521\u2013527)."},{"key":"9313_CR33","unstructured":"Kouylekov, M., Dini, L., Bosca, A., & Trevisan, M. (2013). Celi: EDITS and generic text pair classification. In Second joint conference on lexical and computational semantics (*SEM), Vol. 2: Proceedings of the seventh international workshop on semantic evaluation (SemEval 2013) (pp. 592\u2013597). Atlanta, GA: Association for Computational Linguistics."},{"issue":"4","key":"9313_CR34","doi-asserted-by":"publisher","first-page":"389","DOI":"10.1023\/A:1025779619903","volume":"37","author":"C Leacock","year":"2003","unstructured":"Leacock, C., & Chodorow, M. (2003). C-rater: Automated scoring of short-answer questions. Computers and the Humanities, 37(4), 389\u2013405.","journal-title":"Computers and the Humanities"},{"key":"9313_CR35","doi-asserted-by":"crossref","unstructured":"Leacock, C., Chodorow, M., Gamon, M., & Tetreault, J. R. (2014). Automated grammatical error detection for language learners, Second edition. Synthesis lectures on human language technologies. San Rafael: Morgan & Claypool.","DOI":"10.1007\/978-3-031-02153-4"},{"key":"9313_CR36","unstructured":"Levy, O., Zesch, T., Dagan, I., & Gurevych, I. (2013). UKP-BIU: Similarity and entailment metrics for student response analysis. In Second joint conference on lexical and computational semantics (*SEM), Vol. 2: Proceedings of the seventh international workshop on semantic evaluation (SemEval 2013) (pp. 285\u2013289). Atlanta, GA: Association for Computational Linguistics."},{"key":"9313_CR37","doi-asserted-by":"publisher","first-page":"105","DOI":"10.1109\/TCOM.1982.1095380","volume":"30","author":"NH MacDonald","year":"1982","unstructured":"MacDonald, N. H., Frase, L. T., Gingrich, P. S., & Keenan, S. A. (1982). The writer\u2019s workbench: Computer aids for text analysis. IEEE Transactions on Communications, 30, 105\u2013110.","journal-title":"IEEE Transactions on Communications"},{"key":"9313_CR38","unstructured":"McConville, M., & Dzikovska, M. O. (2008). Deep grammatical relations for semantic interpretation. In Coling 2008: Proceedings of the workshop on cross-framework and cross-domain parser evaluation (pp. 51\u201358)."},{"key":"9313_CR39","unstructured":"Mohler, M., Bunescu, R., & Mihalcea, R. (2011). Learning to grade short answer questions using semantic similarity measures and dependency graph alignments. In Proceedings of the 49th annual meeting of the association for computational linguistics: Human language technologies. Portland, OR: Association for Computational Linguistics (pp. 752\u2013762). http:\/\/www.aclweb.org\/anthology\/P11-1076 ."},{"key":"9313_CR41","unstructured":"Ng, H. T., Wu, S. M., Briscoe, T., Hadiwinoto, C., Susanto, R. H., & Bryant, C. (2014). The CoNLL-201r shared task on grammatical error correction. In Proceedings of the 18th conference on computational natural language learning: Shared task (pp. 1\u201314). Association for Computational Linguistics."},{"key":"9313_CR40","unstructured":"Ng, H. T., Wu, S. M., Wu, Y., & Tetreault, J. (2013). The CoNLL-2013 shared task on grammatical error correction. In Proceedings of the 17th conference on computational natural language learning. Association for Computational Linguistics."},{"key":"9313_CR42","unstructured":"Nielsen, R. D., Ward, W., & Martin, J. H. (2008a). Learning to assess low-level conceptual understanding. In Proceedings of 21st international FLAIRS conference (pp. 427\u2013432)."},{"key":"9313_CR43","unstructured":"Nielsen, R. D., Ward, W., Martin, J. H., & Palmer, M. (2008b). Annotating students\u2019 understanding of science concepts. In Proceedings of the sixth international language resources and evaluation conference (LREC08), Marrakech."},{"key":"9313_CR44","doi-asserted-by":"publisher","first-page":"479","DOI":"10.1017\/S135132490999012X","volume":"15","author":"RD Nielsen","year":"2009","unstructured":"Nielsen, R. D., Ward, W., & Martin, J. H. (2009). Recognizing entailment in intelligent tutoring systems. The Journal of Natural Language Engineering, 15, 479\u2013501.","journal-title":"The Journal of Natural Language Engineering"},{"key":"9313_CR45","unstructured":"Okoye, I., Bethard, S., & Sumner, T. (2013). CU: Computational assessment of short free text answers\u2014A tool for evaluating students\u2019 understanding. In Second joint conference on lexical and computational semantics (*SEM), Vol. 2: Proceedings of the seventh international workshop on semantic evaluation (SemEval 2013) (pp. 603\u2013607). Atlanta, GA: Association for Computational Linguistics."},{"key":"9313_CR46","unstructured":"Ott, N., Ziai, R., Hahn, M., & Meurers, D. (2013). CoMeT: Integrating different levels of linguistic modeling for meaning assessment. In Second joint conference on lexical and computational semantics (*SEM), Vol. 2: Proceedings of the seventh international workshop on semantic evaluation (SemEval 2013) (pp. 608\u2013616). Atlanta, GA: Association for Computational Linguistics."},{"key":"9313_CR47","volume-title":"The imminence of grading essays by computer","author":"E Page","year":"1996","unstructured":"Page, E. (1996). The imminence of grading essays by computer. Bloomington: Phi Delta Kappa."},{"key":"9313_CR48","doi-asserted-by":"crossref","unstructured":"Pon-Barry, H., Clark, B., Schultz, K., Bratt, E. O., & Peters, S. (2004). Advantages of spoken language interaction in dialogue-based intelligent tutoring systems. In Proceedings of ITS-2004 conference (pp. 390\u2013400).","DOI":"10.1007\/978-3-540-30139-4_37"},{"key":"9313_CR49","unstructured":"Pulman, S. G., & Sukkarieh, J. Z. (2005). Automatic short answer marking. In Proceedings of the second workshop on building educational applications using NLP (pp. 9\u201316). Ann Arbor, MI: Association for Computational Linguistics."},{"key":"9313_CR50","volume-title":"Handbook on automated essay evaluation: Current applications and new directions","year":"2013","unstructured":"Shermis, M. D., & Burstein, J. (Eds.). (2013). Handbook on automated essay evaluation: Current applications and new directions. London: Routledge."},{"key":"9313_CR51","unstructured":"Tetreault, J., Blanchard, D., & Cahill, A. (2013). 12:10 a report on the first native language identification shared task. In Proceedings of the eighth workshop on innovative use of NLP for building educational applications (pp. 48\u201357). Association for Computational Linguistics."},{"key":"9313_CR52","doi-asserted-by":"crossref","unstructured":"VanLehn, K., Jordan, P., & Litman, D. (2007). Developing pedagogically effective tutorial dialogue tactics: Experiments and a testbed. In Proceedings of SLaTE workshop on speech and language technology in education. Farmington, PA.","DOI":"10.21437\/SLaTE.2007-3"},{"key":"9313_CR53","doi-asserted-by":"crossref","unstructured":"Wolska, M., & Kruijff-Korbayov\u00e1, I. (2004) Analysis of mixed natural and symbolic language input in mathematical dialogs. In ACL-2004, Barcelona.","DOI":"10.3115\/1218955.1218959"},{"key":"9313_CR54","unstructured":"Yeh, A. (2000). More accurate tests for the statistical significance of result differences. In Proceedings of the 18th international conference on computational linguistics (COLING 2000) (pp. 947\u2013953). Stroudsburg, PA: Association for Computational Linguistics."}],"container-title":["Language Resources and Evaluation"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10579-015-9313-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10579-015-9313-8\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10579-015-9313-8","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10579-015-9313-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,6,10]],"date-time":"2024-06-10T13:48:50Z","timestamp":1718027330000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10579-015-9313-8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,8,13]]},"references-count":53,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2016,3]]}},"alternative-id":["9313"],"URL":"https:\/\/doi.org\/10.1007\/s10579-015-9313-8","relation":{},"ISSN":["1574-020X","1574-0218"],"issn-type":[{"value":"1574-020X","type":"print"},{"value":"1574-0218","type":"electronic"}],"subject":[],"published":{"date-parts":[[2015,8,13]]},"assertion":[{"value":"13 August 2015","order":1,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}