{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,20]],"date-time":"2026-06-20T04:52:56Z","timestamp":1781931176912,"version":"3.54.5"},"reference-count":38,"publisher":"Association for Natural Language Processing","issue":"2","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Journal of Natural Language Processing"],"published-print":{"date-parts":[[2026]]},"DOI":"10.5715\/jnlp.33.658","type":"journal-article","created":{"date-parts":[[2026,6,14]],"date-time":"2026-06-14T22:11:38Z","timestamp":1781475098000},"page":"658-690","source":"Crossref","is-referenced-by-count":0,"title":["Design and Analysis of a Mathematics and Safety Tuning Competition for Large Language Models","\u5927\u898f\u6a21\u8a00\u8a9e\u30e2\u30c7\u30eb\u306e\u6570\u5b66\u30fb\u5b89\u5168\u6027\u30c1\u30e5\u30fc\u30cb\u30f3\u30b0\u30b3\u30f3\u30da\u30c6\u30a3\u30b7\u30e7\u30f3\u306e\u8a2d\u8a08\u3068\u5206\u6790"],"prefix":"10.5715","volume":"33","author":[{"given":"Tsuyoshi","family":"Okita","sequence":"first","affiliation":[{"name":"Kyushu Institute of Technology"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Satoru","family":"Katsumata","sequence":"additional","affiliation":[{"name":"Retrieva, Inc."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Keisuke","family":"Kamata","sequence":"additional","affiliation":[{"name":"Weights &amp; Biases"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hirokazu","family":"Kiyomaru","sequence":"additional","affiliation":[{"name":"National Institute of Informatics"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Takashi","family":"Kodama","sequence":"additional","affiliation":[{"name":"National Institute of Informatics"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jun","family":"Suzuki","sequence":"additional","affiliation":[{"name":"Tohoku University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hisami","family":"Suzuki","sequence":"additional","affiliation":[{"name":"National Institute of Informatics"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kouta","family":"Nakayama","sequence":"additional","affiliation":[{"name":"National Institute of Informatics"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Namgi","family":"Han","sequence":"additional","affiliation":[{"name":"The University of Tokyo"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yusuke","family":"Miyao","sequence":"additional","affiliation":[{"name":"The University of Tokyo"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"3685","reference":[{"key":"1","unstructured":"An, B., Zhu, S., Zhang, R., Panaitescu-Liess, M.-A., Xu, Y., and Huang, F. (2024). \u201cAutomatic Pseudo-Harmful Prompt Generation for Evaluating False Refusals in Large Language Models.\u201d  In <i>1st Conference on Language Modeling<\/i>."},{"key":"2","doi-asserted-by":"crossref","unstructured":"\u6709\u5c71\u77e5\u5e0c\uff0c\u9234\u6728\u6f64\uff0c\u9234\u6728\u6b63\u654f\uff0c\u7530\u4e2d\u6dbc\u592a\uff0c\u8d64\u9593\u601c\u5948\uff0c\u897f\u7530\u4eac\u4ecb (2024). \u30af\u30a4\u30ba\u30b3\u30f3\u30da\u30c6\u30a3\u30b7\u30e7\u30f3\u306e\u7d50\u679c\u5206\u6790\u304b\u3089\u898b\u305f \u65e5\u672c\u8a9e\u8cea\u554f\u5fdc\u7b54\u306e\u5230\u9054\u70b9\u3068\u8ab2\u984c.  \u81ea\u7136\u8a00\u8a9e\u51e6\u7406, <b>31<\/b> (1), pp. 47\u201378. [T. Ariyama et al. (2024). Achievements and Challenges in Japanese Question Answering: Insights from Quiz Competition Results, Journal of Natural Language Processing, 2024, 31(1), pp. 47\u201378.].","DOI":"10.5715\/jnlp.31.47"},{"key":"3","unstructured":"Bai, Y., Jones, A., Ndousse, K., Askell, A., Chen, A., DasSarma, N., Drain, D., Fort, S., Ganguli, D., Henighan, T., Joseph, N., Kadavath, S., Kernion, J., Conerly, T., El-Showk, S., Elhage, N., Hatfield-Dodds, Z., Hernandez, D., Hume, T., Johnston, S., Kravec, S., Lovitt, L., Nanda, N., Olsson, C., Amodei, D., Brown, T., Clark, J., McCandlish, S., Olah, C., Mann, B., and Kaplan, J. (2022). \u201cTraining a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback.\u201d  <i>arXiv preprint arXiv:2204.05862<\/i>."},{"key":"4","unstructured":"Cameron, G., Usui, L., Mooney, P., and Howard, A. (2024). \u201cGoogle - Unlock Global Communication with Gemma.\u201d  https:\/\/kaggle.com\/competitions\/gemma-language-tuning."},{"key":"5","doi-asserted-by":"crossref","unstructured":"Dong, G., Yuan, H., Lu, K., Li, C., Xue, M., Liu, D., Wang, W., Yuan, Z., Zhou, C., and Zhou, J. (2024). \u201cHow Abilities in Large Language Models are Affected by Supervised Fine-tuning Data Composition.\u201d  In Ku, L.-W., Martins, A., and Srikumar, V. (Eds.), <i>Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)<\/i>, pp. 177\u2013198, Bangkok, Thailand. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2024.acl-long.12"},{"key":"6","unstructured":"Gemma (2024). \u201cGemma 2: Improving Open Language Models at a Practical Size.\u201d  <i>arXiv preprint arXiv:2408.00118<\/i>."},{"key":"7","unstructured":"Gou, Z., Shao, Z., Gong, Y., Shen, Y., Yang, Y., Huang, M., Duan, N., and Chen, W. (2023). \u201cTora: A Tool-integrated Reasoning Agent for Mathematical Problem Solving.\u201d  <i>arXiv preprint arXiv:2309.17452<\/i>."},{"key":"8","unstructured":"Guo, D., Yang, D., Zhang, H., Song, J., Zhang, R., Xu, R., Zhu, Q., Ma, S., Wang, P., Bi, X., et al. (2025). \u201cDeepseek-r1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning.\u201d  <i>arXiv preprint arXiv:2501.12948<\/i>."},{"key":"9","unstructured":"Hendrycks, D., Burns, C., Kadavath, S., Arora, A., Basart, S., Tang, E., Song, D., and Steinhardt, J. (2021). \u201cMeasuring Mathematical Problem Solving With the MATH Dataset.\u201d  In <i>35th Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)<\/i>."},{"key":"10","unstructured":"\u72e9\u91ce\u82b3\u4f38\uff0c\u7a32\u8449\u901a\u5c06 (2018). \u4eba\u72fc\u77e5\u80fd\u5927\u4f1a\u7b2c\u4e00\u56de\u81ea\u7136\u8a00\u8a9e\u90e8\u9580\u306e\u958b\u50ac.  \u4eba\u5de5\u77e5\u80fd\u5b66\u4f1a\u5168\u56fd\u5927\u4f1a\u8ad6\u6587\u96c6, JSAI2018, pp. 1H2OS13b03\u20131H2OS13b03. [Y. Kano and M. Inaba (2018). Natural Language Task in AI Werewolf Contest, Proceedings of the Annual Conference of JSAI2018, pp. 1H2OS13b03.]."},{"key":"11","unstructured":"\u9ed2\u6fa4\u53cb\u54c9\uff0c\u9ad8\u5c71\u96bc\u77e2\uff0c\u7dbf\u5ca1\u6643\u8f1d\uff0c\u5c0f\u6797\u6ec9\u6cb3\uff0c\u6d45\u539f\u6b63\u5e78\uff0c\u897f\u5185\u6c99\u6075 (2025). \u5927\u898f\u6a21\u8a00\u8a9e\u30e2\u30c7\u30eb\u306e\u305f\u3081\u306e\u65e5\u672c\u8a9e\u5b89\u5168\u6027\u5883\u754c\u30c6\u30b9\u30c8.  \u8a00\u8a9e\u51e6\u7406\u5b66\u4f1a\u7b2c31\u56de\u5e74\u6b21\u5927\u4f1a\u767a\u8868\u8ad6\u6587\u96c6, pp. 1321\u20131326. [T. Kurosawa et al. (2025). Daikibo Gengo Moderu No Tameno Nihongo Anzensei Kyokai Tesuto. Proceedings of the 31st Annual Meeting of the Association for Natural Language Processing, pp. 1321\u20131326.]."},{"key":"12","unstructured":"LI, J., Beeching, E., Tunstall, L., Lipkin, B., Soletskyi, R., Huang, S. C., Rasul, K., Yu, L., Jiang, A., Shen, Z., Qin, Z., Dong, B., Zhou, L., Fleureau, Y., Lample, G., and Polu, S. (2024a). \u201cNuminaMath CoT.\u201d  [https:\/\/huggingface.co\/AI-MO\/NuminaMath-CoT](https:\/\/github.com\/project-numina\/aimo-progress-prize\/blob\/main\/report\/numina_dataset.pdf)."},{"key":"13","unstructured":"LI, J., Beeching, E., Tunstall, L., Lipkin, B., Soletskyi, R., Huang, S. C., Rasul, K., Yu, L., Jiang, A., Shen, Z., Qin, Z., Dong, B., Zhou, L., Fleureau, Y., Lample, G., and Polu, S. (2024b). \u201cNuminaMath TIR.\u201d  [https:\/\/huggingface.co\/AI-MO\/NuminaMath-TIR](https:\/\/github.com\/project-numina\/aimo-progress-prize\/blob\/main\/report\/numina_dataset.pdf)."},{"key":"14","unstructured":"Liang, P., Bommasani, R., Lee, T., Tsipras, D., Soylu, D., Yasunaga, M., Zhang, Y., Narayanan, D., Wu, Y., Kumar, A., Newman, B., Yuan, B., Yan, B., Zhang, C., Cosgrove, C., Manning, C. D., Re, C., Acosta-Navas, D., Hudson, D. A., Zelikman, E., Durmus, E., Ladhak, F., Rong, F., Ren, H., Yao, H., WANG, J., Santhanam, K., Orr, L., Zheng, L., Yuksekgonul, M., Suzgun, M., Kim, N., Guha, N., Chatterji, N. S., Khattab, O., Henderson, P., Huang, Q., Chi, R. A., Xie, S. M., Santurkar, S., Ganguli, S., Hashimoto, T., Icard, T., Zhang, T., Chaudhary, V., Wang, W., Li, X., Mai, Y., Zhang, Y., and Koreeda, Y. (2023). \u201cHolistic Evaluation of Language Models.\u201d  <i>Transactions on Machine Learning Research<\/i>. Featured Certification, Expert Certification, Outstanding Certification."},{"key":"15","unstructured":"Miyao, Y., Ishida, S., Okamoto, T., Han, N., Mousterou, A., Fourrier, C., Hayashi, T., and Tachibana, Y. (2024). \u201cOpen Japanese LLM Leaderboard.\u201d  https:\/\/huggingface.co\/spaces\/llm-jp\/open-japanese-llm-leaderboard."},{"key":"16","unstructured":"Nakazawa, T., Mino, H., Goto, I., Kurohashi, S., and Sumita, E. (2014). \u201cOverview of the 1st Workshop on Asian Translation.\u201d  In Nakazawa, T., Mino, H., Goto, I., Kurohashi, S., and Sumita, E. (Eds.), <i>Proceedings of the 1st Workshop on Asian Translation (WAT2014)<\/i>, pp. 1\u201319, Tokyo, Japan. Workshop on Asian Translation."},{"key":"17","unstructured":"Ning, Z., Gu, T., Song, J., Hong, S., Li, L., Liu, H., Li, J., Wang, Y., Lingyu, M., Teng, Y., and Wang, Y. (2025). \u201cLinguaSafe: A Comprehensive Multilingual Safety Benchmark for Large Language Models.\u201d  <i>arXiv preprint arXiv:2508.12733<\/i>."},{"key":"18","unstructured":"Ogawa, Y., Kimura, Y., Shibuki, H., Ototake, H., Uchida, Y., Takamaru, K., Kadowaki, K., Akiba, T., Sasaki, M., Kobayashi, A., Yoshioka, M., Mori, T., Araki, K., Sekine, S., and Mitamura, T. (2023). \u201cOverview of the NTCIR-17 QA Lab-PoliInfo-4 Task.\u201d  In <i>Proceedings of NTCIR-17 Conference<\/i>, pp. 217\u2013231."},{"key":"19","doi-asserted-by":"crossref","unstructured":"Rafailov, R., Sharma, A., Mitchell, E., Manning, C. D., Ermon, S., and Finn, C. (2023). \u201cDirect Preference Optimization: Your Language Model is Secretly a Reward Model.\u201d  In <i>37th Conference on Neural Information Processing Systems<\/i>.","DOI":"10.52202\/075280-2338"},{"key":"20","unstructured":"Saroufim, M., Perlitz, Y., Choshen, L., Antiga, L., Bowyer, G., Puhrsch, C., Guessous, D., Rao, S., Chauhan, G., Kumar, A., et al. (2025). \u201cNeurIPS 2023 LLM Efficiency Fine-tuning Competition.\u201d  <i>arXiv preprint arXiv:2503.13507<\/i>."},{"key":"21","doi-asserted-by":"crossref","unstructured":"Sekine, S., and Isahara, H. (2000). \u201cIREX: IR &amp; IE Evaluation Project in Japanese.\u201d  In Gavrilidou, M., Carayannis, G., Markantonatou, S., Piperidis, S., and Stainhauer, G. (Eds.), <i>Proceedings of the 2nd International Conference on Language Resources and Evaluation (LREC\u201900)<\/i>, Athens, Greece. European Language Resources Association (ELRA).","DOI":"10.63317\/5gapt594kwyb"},{"key":"22","unstructured":"Shao, Z., Wang, P., Zhu, Q., Xu, R., Song, J., Bi, X., Zhang, H., Zhang, M., Li, Y., Wu, Y., et al. (2024). \u201cDeepseekmath: Pushing the Limits of Mathematical Reasoning in Open Language Models.\u201d  <i>arXiv preprint arXiv:2402.03300<\/i>."},{"key":"23","unstructured":"Shi, F., Suzgun, M., Freitag, M., Wang, X., Srivats, S., Vosoughi, S., Chung, H. W., Tay, Y., Ruder, S., Zhou, D., Das, D., and Wei, J. (2023). \u201cLanguage Models are Multilingual Chain-of-thought Reasoners.\u201d  In <i>The 11th International Conference on Learning Representations<\/i>."},{"key":"24","doi-asserted-by":"crossref","unstructured":"Suzuki, H., Katsumata, S., Kodama, T., Takahashi, T., Nakayama, K., and Sekine, S. (2025). \u201cAnswerCarefully: A Dataset for Improving the Safety of Japanese LLM Output.\u201d  <i>arXiv preprint arXiv:2506.02372<\/i>.","DOI":"10.63317\/289xxmwcworn"},{"key":"25","doi-asserted-by":"crossref","unstructured":"Wang, W., Tu, Z., Chen, C., Yuan, Y., Huang, J.-t., Jiao, W., and Lyu, M. (2024). \u201cAll Languages Matter: On the Multilingual Safety of LLMs.\u201d  In Ku, L.-W., Martins, A., and Srikumar, V. (Eds.), <i>Findings of the Association for Computational Linguistics: ACL 2024<\/i>, pp. 5865\u20135877, Bangkok, Thailand. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2024.findings-acl.349"},{"key":"26","unstructured":"Wang, X., Wei, J., Schuurmans, D., Le, Q. V., Chi, E. H., Narang, S., Chowdhery, A., and Zhou, D. (2023). \u201cSelf-Consistency Improves Chain of Thought Reasoning in Language Models.\u201d  In <i>The 11th International Conference on Learning Representations<\/i>."},{"key":"27","doi-asserted-by":"crossref","unstructured":"Wang, Y., Li, H., Han, X., Nakov, P., and Baldwin, T. (2024a). \u201cDo-Not-Answer: Evaluating Safeguards in LLMs.\u201d  In Graham, Y., and Purver, M. (Eds.), <i>Findings of the Association for Computational Linguistics: EACL 2024<\/i>, pp. 896\u2013911, St. Julian\u2019s, Malta. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2024.findings-eacl.61"},{"key":"28","doi-asserted-by":"crossref","unstructured":"Wang, Y., Zhai, Z., Li, H., Han, X., Lin, S., Zhang, Z., Zhao, A., Nakov, P., and Baldwin, T. (2024b). \u201cA Chinese Dataset for Evaluating the Safeguards in Large Language Models.\u201d  In Ku, L.-W., Martins, A., and Srikumar, V. (Eds.), <i>Findings of the Association for Computational Linguistics: ACL 2024<\/i>, pp. 3106\u20133119, Bangkok, Thailand. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2024.findings-acl.184"},{"key":"29","unstructured":"Wang, Z., Zhou, F., Li, X., and Liu, P. (2025). \u201cOctoThinker: Mid-training Incentivizes Reinforcement Learning Scaling.\u201d  <i>arXiv preprint arXiv:2506.20512<\/i>."},{"key":"30","doi-asserted-by":"crossref","unstructured":"Wei, J., Wang, X., Schuurmans, D., Bosma, M., Xia, F., Chi, E., Le, Q. V., Zhou, D., et al. (2022). \u201cChain-of-thought Prompting Elicits Reasoning in Large Language Models.\u201d  <i>Advances in Neural Information Processing Systems<\/i>, <b>35<\/b>, pp. 24824\u201324837.","DOI":"10.52202\/068431-1800"},{"key":"31","doi-asserted-by":"crossref","unstructured":"Yadav, P., Tam, D., Choshen, L., Raffel, C., and Bansal, M. (2023). \u201cTIES-Merging: Resolving Interference When Merging Models.\u201d  In <i>37th Conference on Neural Information Processing Systems<\/i>.","DOI":"10.52202\/075280-0310"},{"key":"32","unstructured":"Yan, J., Li, Y., Hu, Z., Wang, Z., Cui, G., Qu, X., Cheng, Y., and Zhang, Y. (2025). \u201cLearning to Reason under Off-Policy Guidance.\u201d  In <i>Proceedings of the 39th Annual Conference on Neural Information Processing Systems<\/i>."},{"key":"33","unstructured":"Yang, A., Zhang, B., Hui, B., Gao, B., Yu, B., Li, C., Liu, D., Tu, J., Zhou, J., Lin, J., Lu, K., Xue, M., Lin, R., Liu, T., Ren, X., and Zhang, Z. (2024a). \u201cQwen2.5-Math Technical Report: Toward Mathematical Expert Model via Self-Improvement.\u201d  <i>arXiv preprint arXiv:2409.12122<\/i>."},{"key":"34","unstructured":"Yang, W., Saroufim, M., Issacson, J., Antiga, L., Bowyer, G., Puhrsch, C., Guessous, D., Rao, S., Chauhan, G., Li, M., Harmeyer, D., and May, W. (2024b). \u201cHAC: The Hacker-Cup AI Competition.\u201d  In <i>NeurIPS 2024 Competition Track<\/i>."},{"key":"35","unstructured":"Yu, L., Yu, B., Yu, H., Huang, F., and Li, Y. (2024). \u201cLanguage Models are Super Mario: Absorbing Abilities from Homologous Models as a Free Lunch.\u201d  In <i>Proceedings of the 41st International Conference on Machine Learning<\/i>, ICML\u201924. JMLR.org."},{"key":"36","unstructured":"Zhang, Z., Xu, W., Wu, F., and Reddy, C. K. (2025). \u201cFalseReject: A Resource for Improving Contextual Safety and Mitigating Over-Refusals in LLMs via Structured Reasoning.\u201d  In <i>2nd Conference on Language Modeling<\/i>."},{"key":"37","unstructured":"Zhao, R., Meterez, A., Kakade, S. M., Pehlevan, C., Jelassi, S., and Malach, E. (2025). \u201cEcho Chamber: RL Post-training Amplifies Behaviors Learned in Pretraining.\u201d  In <i>Proceedings of the 2nd Conference on Language Modeling<\/i>."},{"key":"38","doi-asserted-by":"crossref","unstructured":"Zheng, L., Chiang, W.-L., Sheng, Y., Zhuang, S., Wu, Z., Zhuang, Y., Lin, Z., Li, Z., Li, D., Xing, E., Zhang, H., Gonzalez, J. E., and Stoica, I. (2023). \u201cJudging LLM-as-a-Judge with MT-Bench and Chatbot Arena.\u201d  In Oh, A., Naumann, T., Globerson, A., Saenko, K., Hardt, M., and Levine, S. (Eds.), <i>Advances in Neural Information Processing Systems<\/i>, Vol. 36, pp. 46595\u201346623. Curran Associates, Inc.","DOI":"10.52202\/075280-2020"}],"container-title":["Journal of Natural Language Processing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.jstage.jst.go.jp\/article\/jnlp\/33\/2\/33_658\/_pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,20]],"date-time":"2026-06-20T04:44:20Z","timestamp":1781930660000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.jstage.jst.go.jp\/article\/jnlp\/33\/2\/33_658\/_article\/-char\/ja\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"references-count":38,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2026]]}},"URL":"https:\/\/doi.org\/10.5715\/jnlp.33.658","relation":{},"ISSN":["1340-7619","2185-8314"],"issn-type":[{"value":"1340-7619","type":"print"},{"value":"2185-8314","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]}}}