{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,25]],"date-time":"2026-01-25T03:04:03Z","timestamp":1769310243427,"version":"3.49.0"},"reference-count":31,"publisher":"Association for Natural Language Processing","issue":"2","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Journal of Natural Language Processing"],"published-print":{"date-parts":[[2024]]},"DOI":"10.5715\/jnlp.31.328","type":"journal-article","created":{"date-parts":[[2024,6,14]],"date-time":"2024-06-14T22:13:36Z","timestamp":1718403216000},"page":"328-348","source":"Crossref","is-referenced-by-count":1,"title":["Cloze Quality Estimation for Language Assessment"],"prefix":"10.5715","volume":"31","author":[{"given":"Zizheng","family":"Zhang","sequence":"first","affiliation":[{"name":"Tokyo Metropolitan University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Masato","family":"Mita","sequence":"additional","affiliation":[{"name":"Tokyo Metropolitan University"},{"name":"CyberAgent, Inc."}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mamoru","family":"Komachi","sequence":"additional","affiliation":[{"name":"Tokyo Metropolitan University"},{"name":"Hitotsubashi University"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"3685","reference":[{"key":"1","doi-asserted-by":"crossref","unstructured":"Alderson, J. C. (1979). \u201cThe Cloze Procedure and Proficiency in English as a Foreign Language.\u201d <i>TESOL Quarterly<\/i>, 13, pp. 219\u2013227.","DOI":"10.2307\/3586211"},{"key":"2","unstructured":"ALTE (2011). <i>Manual for Language Test Development and Examining: For Use with the CEFR<\/i>. Language Policy division, Council of Europe."},{"key":"3","doi-asserted-by":"crossref","unstructured":"Aryadoust, V., Zakaria, A., Lim, M. H., and Chen, C. (2020). \u201cAn Extensive Knowledge Mapping Review of Measurement and Validity in Language Assessment and SLA Research.\u201d <i>Frontiers in Psychology<\/i>, 11. 1941.","DOI":"10.3389\/fpsyg.2020.01941"},{"key":"4","doi-asserted-by":"crossref","unstructured":"Bachman, L. F. (1985). \u201cPerformance on Cloze Tests with Fixed-Ratio and Rational Deletions.\u201d <i>TESOL Quarterly<\/i>, 19 (3), pp. 535\u2013556.","DOI":"10.2307\/3586277"},{"key":"5","doi-asserted-by":"crossref","unstructured":"Bryant, C., Felice, M., Andersen, \u00d8. E., and Briscoe, T. (2019). \u201cThe BEA-2019 Shared Task on Grammatical Error Correction.\u201d In <i>Proceedings of the 14th Workshop on Innovative Use of NLP for Building Educational Applications<\/i>, pp. 52\u201375, Florence, Italy. Association for Computational Linguistics.","DOI":"10.18653\/v1\/W19-4406"},{"key":"6","doi-asserted-by":"crossref","unstructured":"Coniam, D. (1997). \u201cA Preliminary Inquiry into Using Corpus Word Frequency Data in the Automatic Generation of English Language Cloze Tests.\u201d <i>Calico Journal<\/i>, 14, pp. 15\u201333.","DOI":"10.1558\/cj.v14i2-4.15-33"},{"key":"7","doi-asserted-by":"crossref","unstructured":"Correia, R., Baptista, J., Eskenazi, M., and Mamede, N. (2012). \u201cAutomatic Generation of Cloze Question Stems.\u201d In <i>International Conference on Computational Processing of the Portuguese Language<\/i>, pp. 168\u2013178. Springer.","DOI":"10.1007\/978-3-642-28885-2_19"},{"key":"8","unstructured":"Dhyaaldian, S. M. A., Al-Zubaidi, S. H., Mutlak, D. A., Neamah, N. R., Albeer, A. A., Hamad, D. A., Al Hasani, S. F., Jaber, M. M., and Maabreh, H. G. (2022). \u201cPsychometric Evaluation of Cloze Tests with the Rasch Model.\u201d <i>International Journal of Language Testing<\/i>, 12 (2), pp. 95\u2013106."},{"key":"9","unstructured":"Felice, M., Taslimipoor, S., Andersen, \u00d8. E., and Buttery, P. (2022). \u201cCEPOC: The Cambridge Exams Publishing Open Cloze dataset.\u201d In <i>Proceedings of the 2022 International Conference on Language Resources and Evaluation<\/i>. European Language Resources Association."},{"key":"10","doi-asserted-by":"crossref","unstructured":"Finn, P. J. (1977). \u201cWord Frequency, Information Theory, and Cloze Performance: A Transfer Feature Theory of Processing in Reading.\u201d <i>Reading Research Quarterly<\/i>, 13 (4), pp. 508\u2013537.","DOI":"10.2307\/747510"},{"key":"11","doi-asserted-by":"crossref","unstructured":"Fleiss, J. L. (1971). \u201cMeasuring Nominal Scale Agreement among Many Raters.\u201d <i>Psychological Bulletin<\/i>, 76 (5), pp. 378\u2013382.","DOI":"10.1037\/h0031619"},{"key":"12","doi-asserted-by":"crossref","unstructured":"Goto, T., Kojiri, T., Watanabe, T., Iwata, T., and Yamada, T. (2010). \u201cAutomatic Generation System of Multiple-choice Cloze Questions and its Evaluation.\u201d <i>Knowledge Management &amp; E-Learning: An International Journal<\/i>, 2 (3), pp. 210\u2013224.","DOI":"10.34105\/j.kmel.2010.02.016"},{"key":"13","unstructured":"Hill, F., Bordes, A., Chopra, S., and Weston, J. (2016). \u201cThe Goldilocks Principle: Reading Children\u2019s Books with Explicit Memory Representations.\u201d In Bengio, Y. and LeCun, Y. (Eds.), <i>International Conference on Learning Representations (ICLR)<\/i>."},{"key":"14","doi-asserted-by":"crossref","unstructured":"Hill, J. and Simha, R. (2016). \u201cAutomatic Generation of Context-based Fill-in-the-Blank Exercises using Co-occurrence Likelihoods and Google n-grams.\u201d In <i>Proceedings of the 11th Workshop on Innovative Use of NLP for Building Educational Applications<\/i>, pp. 23\u201330.","DOI":"10.18653\/v1\/W16-0503"},{"key":"15","doi-asserted-by":"crossref","unstructured":"Jiang, Z., Xu, F. F., Araki, J., and Neubig, G. (2020). \u201cHow Can We Know What Language Models Know?\u201d <i>Transactions of the Association for Computational Linguistics<\/i>, 8, pp. 423\u2013438.","DOI":"10.1162\/tacl_a_00324"},{"key":"16","doi-asserted-by":"crossref","unstructured":"Klein-Braley, C. (1997). \u201cC-Tests in the Context of Reduced Redundancy Testing: An Appraisal.\u201d <i>Language Testing<\/i>, 14 (1), pp. 47\u201384.","DOI":"10.1177\/026553229701400104"},{"key":"17","unstructured":"Kojima, T., Gu, S. S., Reid, M., Matsuo, Y., and Iwasawa, Y. (2022). \u201cLarge Language Models are Zero-shot Reasoners.\u201d <i>Advances in Neural Information Processing Systems<\/i>, 35, pp. 22199\u201322213."},{"key":"18","doi-asserted-by":"crossref","unstructured":"Kong, X., Gangal, V., and Hovy, E. (2020). \u201cSCDE: Sentence Cloze Dataset with High Quality Distractors From Examinations.\u201d In <i>Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics<\/i>, pp. 5668\u20135683, Online. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2020.acl-main.502"},{"key":"19","unstructured":"Liu, Y., Ott, M., Goyal, N., Du, J., Joshi, M., Chen, D., Levy, O., Lewis, M., Zettlemoyer, L., and Stoyanov, V. (2019). \u201cRoBERTa: A Robustly Optimized BERT Pretraining Approach.\u201d <i>arXiv preprint arXiv:1907.11692<\/i>."},{"key":"20","unstructured":"Miller, G. A. (1998). <i>WordNet: An Electronic Lexical Database<\/i>. MIT Press."},{"key":"21","doi-asserted-by":"crossref","unstructured":"Ng, H. T., Wu, S. M., Briscoe, T., Hadiwinoto, C., Susanto, R. H., and Bryant, C. (2014). \u201cThe CoNLL-2014 Shared Task on Grammatical Error Correction.\u201d In <i>Proceedings of the 18th Conference on Computational Natural Language Learning: Shared Task<\/i>, pp. 1\u201314, Baltimore, Maryland. Association for Computational Linguistics.","DOI":"10.3115\/v1\/W14-1701"},{"key":"22","doi-asserted-by":"crossref","unstructured":"Omelianchuk, K., Atrasevych, V., Chernodub, A., and Skurzhanskyi, O. (2020). \u201cGECToR\u2014Grammatical Error Correction: Tag, Not Rewrite.\u201d In <i>Proceedings of the 15th Workshop on Innovative Use of NLP for Building Educational Applications<\/i>, pp. 163\u2013170, Seattle, WA, USA. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2020.bea-1.16"},{"key":"23","doi-asserted-by":"crossref","unstructured":"Panda, S., Palma Gomez, F., Flor, M., and Rozovskaya, A. (2022). \u201cAutomatic Generation of Distractors for Fill-in-the-Blank Exercises with Round-Trip Neural Machine Translation.\u201d In <i>Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics: Student Research Workshop<\/i>, pp. 391\u2013401, Dublin, Ireland. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2022.acl-srw.31"},{"key":"24","unstructured":"Rankin, E. (1976). \u201cSequence Strategies for Teaching Reading Comprehension with the Cloze Procedure.\u201d In <i>Reading: Theory, Research and Practice, 26th Yearbook of the National Reading Conference<\/i>, pp. 92\u201398, Atlanta, GA. National Reading Conference."},{"key":"25","doi-asserted-by":"crossref","unstructured":"Raymond, P. M. (1988). \u201cCloze Procedure in the Teaching of Reading.\u201d <i>TESL Canada Journal<\/i>, 6, pp. 91\u201397.","DOI":"10.18806\/tesl.v6i1.544"},{"key":"26","unstructured":"Rye, J. (1982). <i>Cloze Procedure and the Teaching of Reading<\/i>. London."},{"key":"27","unstructured":"Sakaguchi, K., Arase, Y., and Komachi, M. (2013). \u201cDiscriminative Approach to Fill-in-the-Blank Quiz Generation for Language Learners.\u201d In <i>Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)<\/i>, pp. 238\u2013242, Sofia, Bulgaria. Association for Computational Linguistics."},{"key":"28","unstructured":"Skory, A. and Eskenazi, M. (2010). \u201cPredicting Cloze Task Quality for Vocabulary Training.\u201d In <i>Proceedings of the NAACL HLT 2010 Fifth Workshop on Innovative Use of NLP for Building Educational Applications<\/i>, pp. 49\u201356, Los Angeles, California. Association for Computational Linguistics."},{"key":"29","doi-asserted-by":"crossref","unstructured":"Taylor, W. L. (1953). \u201c\u201cCloze Procedure\u201d: A New Tool for Measuring Readability.\u201d <i>Journalism Quarterly<\/i>, 30 (4), pp. 415\u2013433.","DOI":"10.1177\/107769905303000401"},{"key":"30","doi-asserted-by":"crossref","unstructured":"Trace, J. (2020). \u201cClozing the Gap: How Far Do Cloze Items Measure?\u201d <i>Language Testing<\/i>, 37 (2), pp. 235\u2013253.","DOI":"10.1177\/0265532219888617"},{"key":"31","doi-asserted-by":"crossref","unstructured":"Xie, Q., Lai, G., Dai, Z., and Hovy, E. (2018). \u201cLarge-scale Cloze Test Dataset Created by Teachers.\u201d In <i>Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing<\/i>, pp. 2344\u20132356, Brussels, Belgium. Association for Computational Linguistics.","DOI":"10.18653\/v1\/D18-1257"}],"container-title":["Journal of Natural Language Processing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.jstage.jst.go.jp\/article\/jnlp\/31\/2\/31_328\/_pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,6,15]],"date-time":"2024-06-15T04:27:45Z","timestamp":1718425665000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.jstage.jst.go.jp\/article\/jnlp\/31\/2\/31_328\/_article"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"references-count":31,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2024]]}},"URL":"https:\/\/doi.org\/10.5715\/jnlp.31.328","relation":{},"ISSN":["1340-7619","2185-8314"],"issn-type":[{"value":"1340-7619","type":"print"},{"value":"2185-8314","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024]]}}}