{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,20]],"date-time":"2025-12-20T05:34:30Z","timestamp":1766208870264,"version":"3.48.0"},"reference-count":46,"publisher":"Association for Natural Language Processing","issue":"4","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Journal of Natural Language Processing"],"published-print":{"date-parts":[[2025]]},"DOI":"10.5715\/jnlp.32.1272","type":"journal-article","created":{"date-parts":[[2025,12,14]],"date-time":"2025-12-14T22:09:54Z","timestamp":1765750194000},"page":"1272-1298","source":"Crossref","is-referenced-by-count":0,"title":["Long-Tail Crisis in Nearest Neighbor Language Models","&lt;i&gt;k&lt;\/i&gt;NN \u8a00\u8a9e\u30e2\u30c7\u30eb\u306e\u4f4e\u983b\u5ea6\u8a9e\u306b\u5bfe\u3059\u308b\u632f\u308b\u821e\u3044\u306e\u5206\u6790"],"prefix":"10.5715","volume":"32","author":[{"given":"Yuto","family":"Nishida","sequence":"first","affiliation":[{"name":"Nara Institute of Science and Technology"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Makoto","family":"Morishita","sequence":"additional","affiliation":[{"name":"Future Corporation"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hiroyuki","family":"Deguchi","sequence":"additional","affiliation":[{"name":"Nara Institute of Science and Technology"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hidetaka","family":"Kamigaito","sequence":"additional","affiliation":[{"name":"Nara Institute of Science and Technology"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Taro","family":"Watanabe","sequence":"additional","affiliation":[{"name":"Nara Institute of Science and Technology"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"3685","reference":[{"key":"1","unstructured":"Alon, U., Xu, F., He, J., Sengupta, S., Roth, D., and Neubig, G. (2022). \u201cNeuro-Symbolic Language Modeling with Automaton-augmented Retrieval.\u201d In Chaudhuri, K., Jegelka, S., Song, L., Szepesvari, C., Niu, G., and Sabato, S. (Eds.), <i>Proceedings of the 39th International Conference on Machine Learning<\/i>, Vol. 162 of <i>Proceedings of Machine Learning Research<\/i>, pp. 468\u2013485. PMLR."},{"key":"2","unstructured":"Asai, A., Wu, Z., Wang, Y., Sil, A., and Hajishirzi, H. (2024a). \u201cSelf-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection.\u201d In <i>The 12th International Conference on Learning Representations<\/i>."},{"key":"3","unstructured":"Asai, A., Zhong, Z., Chen, D., Koh, P. W., Zettlemoyer, L., Hajishirzi, H., and Yih, W.-t. (2024b). \u201cReliable, Adaptable, and Attributable Language Models with Retrieval.\u201d <i>arXiv preprint arXiv:2403.03187<\/i>."},{"key":"4","doi-asserted-by":"crossref","unstructured":"Bhardwaj, R., Polovets, G., and Sunkara, M. (2023). \u201cAdaptation Approaches for Nearest Neighbor Language Models.\u201d In Rogers, A., Boyd-Graber, J., and Okazaki, N. (Eds.), <i>Findings of the Association for Computational Linguistics: ACL 2023<\/i>, pp. 1135\u20131146, Toronto, Canada. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2023.findings-acl.73"},{"key":"5","unstructured":"Borgeaud, S., Mensch, A., Hoffmann, J., Cai, T., Rutherford, E., Millican, K., van den Driessche, G. B., Lespiau, J.-B., Damoc, B., Clark, A., de las Casas, D., Guy, A., Menick, J., Ring, R., Hennigan, T., Huang, S., Maggiore, L., Jones, C., Cassirer, A., Brock, A., Paganini, M., Irving, G., Vinyals, O., Osindero, S., Simonyan, K., Rae, J., Elsen, E., and Sifre, L. (2022). \u201cImproving Language Models by Retrieving from Trillions of Tokens.\u201d In Chaudhuri, K., Jegelka, S., Song, L., Szepesvari, C., Niu, G., and Sabato, S. (Eds.), <i>Proceedings of the 39th International Conference on Machine Learning<\/i>, Vol. 162 of <i>Proceedings of Machine Learning Research<\/i>, pp. 2206\u20132240. PMLR."},{"key":"6","doi-asserted-by":"crossref","unstructured":"Chen, D., Fisch, A., Weston, J., and Bordes, A. (2017). \u201cReading Wikipedia to Answer Open-Domain Questions.\u201d In Barzilay, R. and Kan, M.-Y. (Eds.), <i>Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)<\/i>, pp. 1870\u20131879, Vancouver, Canada. Association for Computational Linguistics.","DOI":"10.18653\/v1\/P17-1171"},{"key":"7","doi-asserted-by":"crossref","unstructured":"Douze, M., Guzhva, A., Deng, C., Johnson, J., Szilvasy, G., Mazar\u00e9, P.-E., Lomeli, M., Hosseini, L., and J\u00e9gou, H. (2024). \u201cThe Faiss library.\u201d <i>arXiv preprint arXiv:2401.08281<\/i>.","DOI":"10.1109\/TBDATA.2025.3618474"},{"key":"8","doi-asserted-by":"crossref","unstructured":"Drozdov, A., Wang, S., Rahimi, R., McCallum, A., Zamani, H., and Iyyer, M. (2022). \u201cYou Can\u2019t Pick Your Neighbors, or Can You? When and How to Rely on Retrieval in the kNN-LM.\u201d In Goldberg, Y., Kozareva, Z., and Zhang, Y. (Eds.), <i>Findings of the Association for Computational Linguistics: EMNLP 2022<\/i>, pp. 2997\u20133007, Abu Dhabi, United Arab Emirates. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2022.findings-emnlp.218"},{"key":"9","unstructured":"Gokaslan, A. and Cohen, V. (2019). \u201cOpenWebText Corpus.\u201d http:\/\/Skylion007.github.io\/OpenWebTextCorpus."},{"key":"10","unstructured":"Grave, E., Cisse, M., and Joulin, A. (2017). \u201cUnbounded Cache Model for Online Language Modeling with Open Vocabulary.\u201d In <i>Proceedings of the 31st International Conference on Neural Information Processing Systems<\/i>, NIPS 2017, pp. 6044\u20136054, Red Hook, NY, USA. Curran Associates Inc."},{"key":"11","doi-asserted-by":"crossref","unstructured":"Guu, K., Hashimoto, T. B., Oren, Y., and Liang, P. (2018). \u201cGenerating Sentences by Editing Prototypes.\u201d <i>Transactions of the Association for Computational Linguistics<\/i>, 6, pp. 437\u2013450.","DOI":"10.1162\/tacl_a_00030"},{"key":"12","doi-asserted-by":"crossref","unstructured":"Hashimoto, W., Kamigaito, H., and Watanabe, T. (2025). \u201cEfficient Nearest Neighbor based Uncertainty Estimation for Natural Language Processing Tasks.\u201d In Chiruzzo, L., Ritter, A., and Wang, L. (Eds.), <i>Findings of the Association for Computational Linguistics: NAACL 2025<\/i>, pp. 4350\u20134366, Albuquerque, New Mexico. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2025.findings-naacl.246"},{"key":"13","unstructured":"He, J., Berg-Kirkpatrick, T., and Neubig, G. (2020). \u201cLearning Sparse Prototypes for Text Generation.\u201d In <i>Proceedings of the 34th International Conference on Neural Information Processing Systems<\/i>, NeurIPS 2020, pp. 14724\u201314735, Red Hook, NY, USA. Curran Associates Inc."},{"key":"14","unstructured":"Hendrycks, D., Burns, C., Basart, S., Zou, A., Mazeika, M., Song, D., and Steinhardt, J. (2021). \u201cMeasuring Massive Multitask Language Understanding.\u201d In <i>International Conference on Learning Representations<\/i>."},{"key":"15","doi-asserted-by":"crossref","unstructured":"J\u00e9gou, H., Douze, M., and Schmid, C. (2011). \u201cProduct Quantization for Nearest Neighbor Search.\u201d <i>IEEE Transactions on Pattern Analysis and Machine Intelligence<\/i>, 33 (1), pp. 117\u2013128.","DOI":"10.1109\/TPAMI.2010.57"},{"key":"16","doi-asserted-by":"crossref","unstructured":"Jiang, H., Lu, Z., Meng, F., Zhou, C., Zhou, J., Huang, D., and Su, J. (2022). \u201cTowards Robust k-Nearest-Neighbor Machine Translation.\u201d In Goldberg, Y., Kozareva, Z., and Zhang, Y. (Eds.), <i>Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing<\/i>, pp. 5468\u20135477, Abu Dhabi, United Arab Emirates. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2022.emnlp-main.367"},{"key":"17","doi-asserted-by":"crossref","unstructured":"Jiang, Q., Wang, M., Cao, J., Cheng, S., Huang, S., and Li, L. (2021). \u201cLearning Kernel-Smoothed Machine Translation with Retrieved Examples.\u201d In Moens, M.-F., Huang, X., Specia, L., and Yih, S. W.-t. (Eds.), <i>Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing<\/i>, pp. 7280\u20137290, Online and Punta Cana, Dominican Republic. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2021.emnlp-main.579"},{"key":"18","doi-asserted-by":"crossref","unstructured":"Johnson, J., Douze, M., and J\u00e9gou, H. (2019). \u201cBillion-scale Similarity Search with GPUs.\u201d <i>IEEE Transactions on Big Data<\/i>, 7 (3), pp. 535\u2013547.","DOI":"10.1109\/TBDATA.2019.2921572"},{"key":"19","unstructured":"Kandpal, N., Deng, H., Roberts, A., Wallace, E., and Raffel, C. (2023). \u201cLarge Language Models Struggle to Learn Long-tail Knowledge.\u201d In <i>Proceedings of the 40th International Conference on Machine Learning<\/i>, pp. 15696\u201315707."},{"key":"20","unstructured":"Khandelwal, U., Fan, A., Jurafsky, D., Zettlemoyer, L., and Lewis, M. (2021). \u201cNearest Neighbor Machine Translation.\u201d In <i>International Conference on Learning Representations<\/i>."},{"key":"21","unstructured":"Khandelwal, U., Levy, O., Jurafsky, D., Zettlemoyer, L., and Lewis, M. (2020). \u201cGeneralization through Memorization: Nearest Neighbor Language Models.\u201d In <i>International Conference on Learning Representations<\/i>."},{"key":"22","unstructured":"Lewis, P., Perez, E., Piktus, A., Petroni, F., Karpukhin, V., Goyal, N., K\u00fcttler, H., Lewis, M., Yih, W.-t., Rockt\u00e4schel, T., Riedel, S., and Kiela, D. (2020). \u201cRetrieval-augmented Generation for Knowledge-intensive NLP Tasks.\u201d In <i>Proceedings of the 34th International Conference on Neural Information Processing Systems<\/i>, NeurIPS 2020, pp. 9459\u20139474, Red Hook, NY, USA. Curran Associates Inc."},{"key":"23","doi-asserted-by":"crossref","unstructured":"Li, K., Povey, D., and Khudanpur, S. (2020). \u201cNeural Language Modeling with Implicit Cache Pointers.\u201d In <i>Proceedings of the 21st Annual Conference of the International Speech Communication Association<\/i>, pp. 3625\u20133629.","DOI":"10.21437\/Interspeech.2020-3020"},{"key":"24","unstructured":"Liu, J., Li, Y., and Wang, H. (2025). \u201cEmergent Specialization: Rare Token Neurons in Language Models.\u201d In <i>Mechanistic Interpretability Workshop at NeurIPS 2025<\/i>."},{"key":"25","unstructured":"Merity, S., Xiong, C., Bradbury, J., and Socher, R. (2017). \u201cPointer Sentinel Mixture Models.\u201d In <i>International Conference on Learning Representations<\/i>."},{"key":"26","unstructured":"Min, S., Gururangan, S., Wallace, E., Shi, W., Hajishirzi, H., Smith, N. A., and Zettlemoyer, L. (2024). \u201cSILO Language Models: Isolating Legal Risk In a Nonparametric Datastore.\u201d In <i>The 12th International Conference on Learning Representations<\/i>."},{"key":"27","doi-asserted-by":"crossref","unstructured":"Min, S., Shi, W., Lewis, M., Chen, X., Yih, W.-t., Hajishirzi, H., and Zettlemoyer, L. (2023). \u201cNonparametric Masked Language Modeling.\u201d In Rogers, A., Boyd-Graber, J., and Okazaki, N. (Eds.), <i>Findings of the Association for Computational Linguistics: ACL 2023<\/i>, pp. 2097\u20132118, Toronto, Canada. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2023.findings-acl.132"},{"key":"28","unstructured":"Nawezi, G., Flek, L., and Welch, C. (2023). \u201cStyle Locality for Controllable Generation with kNN Language Models.\u201d In Hazarika, D., Tang, X. R., and Jin, D. (Eds.), <i>Proceedings of the 1st Workshop on Taming Large Language Models: Controllability in the era of Interactive Assistants!<\/i>, pp. 68\u201375, Prague, Czech Republic. Association for Computational Linguistics."},{"key":"29","doi-asserted-by":"crossref","unstructured":"Nishida, Y., Morishita, M., Deguchi, H., Hidetaka, K., and Taro, W. (2025). \u201cLong-Tail Crisis in Nearest Neighbor Language Models.\u201d In <i>Findings of the Association for Computational Linguistics: NAACL 2025<\/i>, pp. 5965\u20135978. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2025.findings-naacl.331"},{"key":"30","unstructured":"\u897f\u7530\u60a0\u4eba\uff0c\u68ee\u4e0b\u7766\uff0c\u51fa\u53e3\u7965\u4e4b\uff0c\u4e0a\u57a3\u5916\u82f1\u525b\uff0c\u6e21\u8fba\u592a\u90ce (2024). kNN \u8a00\u8a9e\u30e2\u30c7\u30eb\u306f\u4f4e\u983b\u5ea6\u8a9e\u306e\u4e88\u6e2c\u306b\u5f79\u7acb\u3064\u304b\uff1f \u8a00\u8a9e\u51e6\u7406\u5b66\u4f1a\u7b2c30\u56de\u5e74\u6b21\u5927\u4f1a\u767a\u8868\u8ad6\u6587\u96c6, pp. 3209\u20133214. [Y. Nishida et al. (2024). kNN Gengo Moderu wa Teihindo-go no Yosoku ni Yakudatsu ka? Gengoshorigakkai Dai 30kai Nenjitaikai Happyo Ronbunshu, pp. 3209\u20133214.]."},{"key":"31","unstructured":"Pal, A., Umapathi, L. K., and Sankarasubbu, M. (2022). \u201cMedMCQA: A Large-scale Multi-Subject Multi-Choice Dataset for Medical Domain Question Answering.\u201d In Flores, G., Chen, G. H., Pollard, T., Ho, J. C., and Naumann, T. (Eds.), <i>Proceedings of the Conference on Health, Inference, and Learning<\/i>, Vol. 174 of <i>Proceedings of Machine Learning Research<\/i>, pp. 248\u2013260. PMLR."},{"key":"32","unstructured":"Pinto, A., Galanti, T., and Balestriero, R. (2024). \u201cThe Fair Language Model Paradox.\u201d <i>arXiv preprint arXiv:2410.11985<\/i>."},{"key":"33","unstructured":"Radford, A., Wu, J., Child, R., Luan, D., Amodei, D., and Sutskever, I. (2019). \u201cLanguage Models are Unsupervised Multitask Learners.\u201d <i>OpenAI blog<\/i>, 1 (8), p. 9."},{"key":"34","unstructured":"Robinson, J. and Wingate, D. (2023). \u201cLeveraging Large Language Models for Multiple Choice Question Answering.\u201d In <i>The 11th International Conference on Learning Representations<\/i>."},{"key":"35","doi-asserted-by":"crossref","unstructured":"Shi, W., Michael, J., Gururangan, S., and Zettlemoyer, L. (2022). \u201cNearest Neighbor Zero-Shot Inference.\u201d In Goldberg, Y., Kozareva, Z., and Zhang, Y. (Eds.), <i>Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing<\/i>, pp. 3254\u20133265, Abu Dhabi, United Arab Emirates. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2022.emnlp-main.214"},{"key":"36","doi-asserted-by":"crossref","unstructured":"Su, Z., Lin, Z., Baixue, B., Chen, H., Hu, S., Zhou, W., Ding, G., and W, X. (2024). \u201cMiLe Loss: A New Loss for Mitigating the Bias of Learning Difficulties in Generative Language Models.\u201d In Duh, K., Gomez, H., and Bethard, S. (Eds.), <i>Findings of the Association for Computational Linguistics: NAACL 2024<\/i>, pp. 250\u2013262, Mexico City, Mexico. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2024.findings-naacl.18"},{"key":"37","doi-asserted-by":"crossref","unstructured":"Trotta, S., Flek, L., and Welch, C. (2022). \u201cNearest Neighbor Language Models for Stylistic Controllable Generation.\u201d In Bosselut, A., Chandu, K., Dhole, K., Gangal, V., Gehrmann, S., Jernite, Y., Novikova, J., and Perez-Beltrachini, L. (Eds.), <i>Proceedings of the 2nd Workshop on Natural Language Generation, Evaluation, and Metrics (GEM)<\/i>, pp. 295\u2013305, Abu Dhabi, United Arab Emirates (Hybrid). Association for Computational Linguistics.","DOI":"10.18653\/v1\/2022.gem-1.25"},{"key":"38","unstructured":"Wang, B., Ping, W., McAfee, L., Xu, P., Li, B., Shoeybi, M., and Catanzaro, B. (2024). \u201cInstructRetro: Instruction Tuning post Retrieval-Augmented Pretraining.\u201d In <i>The 12th International Conference on Learning Representations<\/i>."},{"key":"39","doi-asserted-by":"crossref","unstructured":"Wang, B., Ping, W., Xu, P., McAfee, L., Liu, Z., Shoeybi, M., Dong, Y., Kuchaiev, O., Li, B., Xiao, C., Anandkumar, A., and Catanzaro, B. (2023a). \u201cShall We Pretrain Autoregressive Language Models with Retrieval? A Comprehensive Study.\u201d In Bouamor, H., Pino, J., and Bali, K. (Eds.), <i>Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing<\/i>, pp. 7763\u20137786, Singapore. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2023.emnlp-main.482"},{"key":"40","doi-asserted-by":"crossref","unstructured":"Wang, S., Song, Y., Drozdov, A., Garimella, A., Manjunatha, V., and Iyyer, M. (2023b). \u201c<i>k<\/i>NN-LM Does Not Improve Open-ended Text Generation.\u201d In Bouamor, H., Pino, J., and Bali, K. (Eds.), <i>Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing<\/i>, pp. 15023\u201315037, Singapore. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2023.emnlp-main.929"},{"key":"41","unstructured":"Xu, F. F., Alon, U., and Neubig, G. (2023). \u201cWhy Do Nearest Neighbor Language Models Work?\u201d In <i>Proceedings of the 40th International Conference on Machine Learning<\/i>, ICML\u201923, pp. 38325\u201338341. JMLR.org."},{"key":"42","unstructured":"Yao, S., Zhao, J., Yu, D., Du, N., Shafran, I., Narasimhan, K. R., and Cao, Y. (2023). \u201cReAct: Synergizing Reasoning and Acting in Language Models.\u201d In <i>The 11th International Conference on Learning Representations<\/i>."},{"key":"43","doi-asserted-by":"crossref","unstructured":"Yokoi, S., Bao, H., Kurita, H., and Shimodaira, H. (2024). \u201cZipfian Whitening.\u201d In Globerson, A., Mackey, L., Belgrave, D., Fan, A., Paquet, U., Tomczak, J., and Zhang, C. (Eds.), <i>Advances in Neural Information Processing Systems<\/i>, Vol. 37, pp. 122259\u2013122291. Curran Associates, Inc.","DOI":"10.52202\/079017-3885"},{"key":"44","doi-asserted-by":"crossref","unstructured":"Zheng, X., Zhang, Z., Guo, J., Huang, S., Chen, B., Luo, W., and Chen, J. (2021). \u201cAdaptive Nearest Neighbor Machine Translation.\u201d In Zong, C., Xia, F., Li, W., and Navigli, R. (Eds.), <i>Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 2: Short Papers)<\/i>, pp. 368\u2013374, Online. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2021.acl-short.47"},{"key":"45","doi-asserted-by":"crossref","unstructured":"Zhong, Z., Lei, T., and Chen, D. (2022). \u201cTraining Language Models with Memory Augmentation.\u201d In Goldberg, Y., Kozareva, Z., and Zhang, Y. (Eds.), <i>Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing<\/i>, pp. 5657\u20135673, Abu Dhabi, United Arab Emirates. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2022.emnlp-main.382"},{"key":"46","doi-asserted-by":"crossref","unstructured":"Zong, Q., Wang, Z., Zheng, T., Ren, X., and Song, Y. (2025). \u201cComparisonQA: Evaluating Factuality Robustness of LLMs Through Knowledge Frequency Control and Uncertainty.\u201d In Che, W., Nabende, J., Shutova, E., and Pilehvar, M. T. (Eds.), <i>Findings of the Association for Computational Linguistics: ACL 2025<\/i>, pp. 4101\u20134117, Vienna, Austria. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2025.findings-acl.212"}],"container-title":["Journal of Natural Language Processing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.jstage.jst.go.jp\/article\/jnlp\/32\/4\/32_1272\/_pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,20]],"date-time":"2025-12-20T03:39:39Z","timestamp":1766201979000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.jstage.jst.go.jp\/article\/jnlp\/32\/4\/32_1272\/_article\/-char\/ja\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"references-count":46,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2025]]}},"URL":"https:\/\/doi.org\/10.5715\/jnlp.32.1272","relation":{},"ISSN":["1340-7619","2185-8314"],"issn-type":[{"type":"print","value":"1340-7619"},{"type":"electronic","value":"2185-8314"}],"subject":[],"published":{"date-parts":[[2025]]}}}