{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,16]],"date-time":"2026-06-16T12:26:48Z","timestamp":1781612808849,"version":"3.54.5"},"reference-count":67,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,7,1]],"date-time":"2026-07-01T00:00:00Z","timestamp":1782864000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,7,1]],"date-time":"2026-07-01T00:00:00Z","timestamp":1782864000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,7,1]],"date-time":"2026-07-01T00:00:00Z","timestamp":1782864000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-017"},{"start":{"date-parts":[[2026,7,1]],"date-time":"2026-07-01T00:00:00Z","timestamp":1782864000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"},{"start":{"date-parts":[[2026,7,1]],"date-time":"2026-07-01T00:00:00Z","timestamp":1782864000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-012"},{"start":{"date-parts":[[2026,7,1]],"date-time":"2026-07-01T00:00:00Z","timestamp":1782864000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,7,1]],"date-time":"2026-07-01T00:00:00Z","timestamp":1782864000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-004"}],"funder":[{"DOI":"10.13039\/501100003407","name":"Ministero dell\u2019Istruzione, dell\u2019Universit\u00e0 e della Ricerca","doi-asserted-by":"publisher","award":["H53D23003510006"],"award-info":[{"award-number":["H53D23003510006"]}],"id":[{"id":"10.13039\/501100003407","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003407","name":"Ministero dell\u2019Istruzione, dell\u2019Universit\u00e0 e della Ricerca","doi-asserted-by":"publisher","award":["2022B3BP5S"],"award-info":[{"award-number":["2022B3BP5S"]}],"id":[{"id":"10.13039\/501100003407","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100008510","name":"University of Maryland","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100008510","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100031478","name":"NextGenerationEU","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100031478","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000780","name":"European Commission","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100000780","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100006198","name":"Goddard Space Flight Center","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100006198","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Journal of Systems and Software"],"published-print":{"date-parts":[[2026,7]]},"DOI":"10.1016\/j.jss.2026.112851","type":"journal-article","created":{"date-parts":[[2026,3,11]],"date-time":"2026-03-11T07:44:05Z","timestamp":1773215045000},"page":"112851","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":1,"special_numbering":"C","title":["Issue classification with LLMs: An empirical study of the NASA flight software systems"],"prefix":"10.1016","volume":"237","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3871-401X","authenticated-orcid":false,"given":"Giuseppe","family":"Colavito","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3373-7589","authenticated-orcid":false,"given":"Filippo","family":"Lanubile","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1160-2608","authenticated-orcid":false,"given":"Nicole","family":"Novielli","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Christopher","family":"Arreza","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ying","family":"Shi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.jss.2026.112851_bib0001","series-title":"2023 IEEE\/ACM 10th International Conference on Mobile Software Engineering and Systems (MOBILESoft)","first-page":"40","article-title":"Issue-labeler: an ALBERT-based jira plugin for issue classification","author":"Alhindi","year":"2023"},{"key":"10.1016\/j.jss.2026.112851_bib0002","series-title":"48th IEEE\/ACM International Conference on Software Engineering, ICSE 2026","first-page":"[TOAPPEAR]","article-title":"Reflections on the reproducibility of commercial LLM performance in empirical software engineering studies","author":"Angermeir","year":"2026"},{"key":"10.1016\/j.jss.2026.112851_bib0003","unstructured":"Anthropic, 2024. Claude 3 family. Accessed: 2024-09-17. https:\/\/www.anthropic.com\/news\/claude-3-family."},{"key":"10.1016\/j.jss.2026.112851_bib0004","series-title":"Proceedings of the 2008 Conf. of the Center for Advanced Studies on Collaborative Research: Meeting of Minds","article-title":"Is it a bug or an enhancement? a text-based approach to classify change requests","author":"Antoniol","year":"2008"},{"key":"10.1016\/j.jss.2026.112851_bib0005","series-title":"Proceedings of the Third ACM\/IEEE International Workshop on NL-Based Software Engineering","first-page":"57-60","article-title":"Applying large language models to issue classification","author":"Aracena","year":"2024"},{"key":"10.1016\/j.jss.2026.112851_bib0006","unstructured":"Bai, J., Bai, S., Chu, Y., Cui, Z., Dang, K., Deng, X., Fan, Y., Ge, W., Han, Y., Huang, F., Hui, B., Ji, L., Li, M., Lin, J., Lin, R., Liu, D., Liu, G., Lu, C., Lu, K., Ma, J., Men, R., Ren, X., Ren, X., Tan, C., Tan, S., Tu, J., Wang, P., Wang, S., Wang, W., Wu, S., Xu, B., Xu, J., Yang, A., Yang, H., Yang, J., Yang, S., Yao, Y., Yu, B., Yuan, H., Yuan, Z., Zhang, J., Zhang, X., Zhang, Y., Zhang, Z., Zhou, C., Zhou, J., Zhou, X., Zhu, T., 2023. Qwen technical report. 2309.16609. arXiv: 2309.16609."},{"key":"10.1016\/j.jss.2026.112851_bib0007","unstructured":"Baltes, S., Angermeir, F., Arora, C., Bar\u00f3n, M. M., Chen, C., B\u00f6hme, L., Calefato, F., Ernst, N., Falessi, D., Fitzgerald, B., Fucci, D., Kalinowski, M., Lambiase, S., Russo, D., Lungu, M., Prechelt, L., Ralph, P., van Tonder, R., Treude, C., Wagner, S., 2025. Guidelines for empirical studies in software engineering involving large language models. Available at https:\/\/arxiv.org\/abs\/2508.15503."},{"key":"10.1016\/j.jss.2026.112851_bib0008","series-title":"2013 IEEE 24th Int\u2019l Symposium on Software Reliability Engineering (ISSRE)","article-title":"Got issues? Who cares about it? A large scale investigation of issue trackers from GitHub","author":"Bissyand\u00e9","year":"2013"},{"key":"10.1016\/j.jss.2026.112851_bib0009","series-title":"Advances in Neural Information Processing Systems","first-page":"1877","article-title":"Language models are few-shot learners","author":"Brown","year":"2020"},{"key":"10.1016\/j.jss.2026.112851_bib0010","unstructured":"Bubeck, S., Chandrasekaran, V., Eldan, R., Gehrke, J., Horvitz, E., Kamar, E., Lee, P., Lee, Y. T., Li, Y., Lundberg, S., Nori, H., Palangi, H., Ribeiro, M. T., Zhang, Y., 2023. Sparks of artificial general intelligence: early experiments with GPT-4. arXiv: 2303.12712."},{"key":"10.1016\/j.jss.2026.112851_bib0011","unstructured":"Cao, J., Li, M., Wen, M., Cheung, S.-c., 2023. A study on prompt design, advantages and limitations of chatGPT for deep learning program repair. arXiv: 2304.08191."},{"key":"10.1016\/j.jss.2026.112851_bib0012","unstructured":"Charalambous, Y., Tihanyi, N., Jain, R., Sun, Y., Ferrag, M. A., Cordeiro, L. C., 2023. A new era in software security: towards self-healing software via large language models and formal verification. arXiv: 2305.14752."},{"key":"10.1016\/j.jss.2026.112851_bib0013","unstructured":"Chen, M., Tworek, J., Jun, H., Yuan, Q., de Oliveira Pinto, H. P., Kaplan, J., Edwards, H., Burda, Y., Joseph, N., Brockman, G., Ray, A., Puri, R., Krueger, G., Petrov, M., Khlaaf, H., Sastry, G., Mishkin, P., Chan, B., Gray, S., Ryder, N., Pavlov, M., Power, A., Kaiser, L., Bavarian, M., Winter, C., Tillet, P., Such, F. P., Cummings, D., Plappert, M., Chantzis, F., Barnes, E., Herbert-Voss, A., Guss, W. H., Nichol, A., Paino, A., Tezak, N., Tang, J., Babuschkin, I., Balaji, S., Jain, S., Saunders, W., Hesse, C., Carr, A. N., Leike, J., Achiam, J., Misra, V., Morikawa, E., Radford, A., Knight, M., Brundage, M., Murati, M., Mayer, K., Welinder, P., McGrew, B., Amodei, D., McCandlish, S., Sutskever, I., Zaremba, W., 2021. Evaluating large language models trained on code. arXiv: 2107.03374."},{"issue":"4","key":"10.1016\/j.jss.2026.112851_bib0014","doi-asserted-by":"crossref","first-page":"213","DOI":"10.1037\/h0026256","article-title":"Weighted kappa: nominal scale agreement provision for scaled disagreement or partial credit","volume":"70","author":"Cohen","year":"1968","journal-title":"Psychol Bull."},{"key":"10.1016\/j.jss.2026.112851_bib0015","series-title":"2023 IEEE\/ACM 2nd International Workshop on Natural Language-Based Software Engineering (NLBSE)","first-page":"16","article-title":"Few-shot learning for issue report classification","author":"Colavito","year":"2023"},{"key":"10.1016\/j.jss.2026.112851_bib0016","series-title":"Proceedings of the 1st International Workshop on Natural Language-Based Software Engineering","first-page":"29-32","article-title":"Issue report classification using pre-trained language models","author":"Colavito","year":"2023"},{"key":"10.1016\/j.jss.2026.112851_bib0017","doi-asserted-by":"crossref","DOI":"10.1016\/j.infsof.2025.107758","article-title":"Benchmarking large language models for automated labeling: the case of issue report classification","volume":"184","author":"Colavito","year":"2025","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.jss.2026.112851_bib0018","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2023.111838","article-title":"Impact of data quality for automatic issue classification using pre-trained language models","volume":"210","author":"Colavito","year":"2024","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.jss.2026.112851_bib0019","series-title":"Proceedings of 21st International Conference on Mining Software Repositories (MSR 2024), April 2024","article-title":"Leveraging GPT-like LLMs to automate issue labeling","author":"Colavito","year":"2024"},{"key":"10.1016\/j.jss.2026.112851_bib0020","series-title":"2015 IEEE 22Nd Int\u2019l Conf.on Software Analysis, Evolution, and Reengineering (SANER)","article-title":"GiLA: github label analyzer","author":"C\u00e1novas Izquierdo","year":"2015"},{"key":"10.1016\/j.jss.2026.112851_bib0021","unstructured":"Dettmers, T., 2021. bitsandbytes. Accessed: 2024-06-17. https:\/\/github.com\/TimDettmers\/bitsandbytes."},{"key":"10.1016\/j.jss.2026.112851_bib0022","series-title":"Advances in Neural Information Processing Systems","first-page":"10088","article-title":"Qlora: efficient finetuning of quantized llms","author":"Dettmers","year":"2023"},{"key":"10.1016\/j.jss.2026.112851_bib0023","series-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)","first-page":"4171","article-title":"BERT: pre-training of deep bidirectional transformers for language understanding","author":"Devlin","year":"2019"},{"key":"10.1016\/j.jss.2026.112851_bib0024","series-title":"2023 IEEE\/ACM International Conference on Software Engineering: Future of Software Engineering (ICSE-FoSE)","first-page":"31","article-title":"Large language models for software engineering: survey and open problems","author":"Fan","year":"2023"},{"key":"10.1016\/j.jss.2026.112851_bib0025","series-title":"2017 ACM\/IEEE Int\u2019l Symposium on Empirical Software Engineering and Measurement (ESEM)","article-title":"Where is the road for issue reports classification based on text mining?","author":"Fan","year":"2017"},{"key":"10.1016\/j.jss.2026.112851_bib0026","unstructured":"GitHub, 2023. Github copilot now has a better AI model and new capabilities. https:\/\/github.blog\/ai-and-ml\/github-copilot\/github-copilot-now-has-a-better-ai-model-and-new-capabilities\/."},{"key":"10.1016\/j.jss.2026.112851_bib0027","unstructured":"He, P., Liu, X., Gao, J., Chen, W., 2021. DeBERTa: decoding-enhanced BERT with disentangled attention. arXiv: 2006.03654."},{"key":"10.1016\/j.jss.2026.112851_bib0028","series-title":"2013 35th International Conference on Software Engineering (ICSE)","first-page":"392","article-title":"It\u2019s not a bug, it\u2019s a feature: how misclassification impacts bug prediction","author":"Herzig","year":"2013"},{"issue":"FSE","key":"10.1016\/j.jss.2026.112851_bib0029","doi-asserted-by":"crossref","DOI":"10.1145\/3660773","article-title":"A deep dive into large language models for automated bug localization and repair","volume":"1","author":"Hossain","year":"2024","journal-title":"Proc. ACM Softw. Eng."},{"issue":"2","key":"10.1016\/j.jss.2026.112851_bib0030","doi-asserted-by":"crossref","DOI":"10.1007\/s10664-021-10085-3","article-title":"Predicting the objective and priority of issue reports in software repositories","author":"Izadi","year":"2022","journal-title":"Empir. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112851_bib0031","unstructured":"Jiang, A. Q., Sablayrolles, A., Mensch, A., Bamford, C., Chaplot, D. S., de las Casas, D., Bressand, F., Lengyel, G., Lample, G., Saulnier, L., Lavaud, L. R., Lachaux, M.-A., Stock, P., Scao, T. L., Lavril, T., Wang, T., Lacroix, T., Sayed, W. E., 2023. Mistral 7b. arXiv: 2310.06825."},{"key":"10.1016\/j.jss.2026.112851_bib0032","series-title":"2019 IEEE Intl. Conf. on Software Maintenance and Evolution, ICSME 2019, Cleveland, OH, USA, September 29, - October 4, 2019","article-title":"Ticket tagger: machine learning driven issue classification","author":"Kallis","year":"2019"},{"key":"10.1016\/j.jss.2026.112851_bib0033","unstructured":"Kaplan, J., McCandlish, S., Henighan, T., Brown, T. B., Chess, B., Child, R., Gray, S., Radford, A., Wu, J., Amodei, D., 2020. Scaling laws for neural language models. arXiv: 2001.08361."},{"key":"10.1016\/j.jss.2026.112851_bib0034","series-title":"2022 IEEE\/ACM International Workshop on Automated Program Repair (APR)","first-page":"61","article-title":"Towards javascript program repair with generative pre-trained transformer (GPT-2)","author":"Lajk\u00f3","year":"2022"},{"key":"10.1016\/j.jss.2026.112851_bib0035","unstructured":"Lan, Z., Chen, M., Goodman, S., Gimpel, K., Sharma, P., Soricut, R., 2020. Albert: a lite bert for self-supervised learning of language representations. arXiv: 1909.11942."},{"key":"10.1016\/j.jss.2026.112851_bib0036","series-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","first-page":"7871","article-title":"BART: denoising sequence-to-sequence pre-training for natural language generation, translation, and comprehension","author":"Lewis","year":"2020"},{"key":"10.1016\/j.jss.2026.112851_bib0037","article-title":"Exploring the characteristics of issue-related behaviors in github using visualization techniques","author":"Liao","year":"2018","journal-title":"IEEE Access"},{"key":"10.1016\/j.jss.2026.112851_bib0038","series-title":"Proceedings of the 2024 IEEE\/ACM First International Conference on AI Foundation Models and Software Engineering","first-page":"57-68","article-title":"Exploring the impact of the output format on the evaluation of large language models for code translation","author":"Macedo","year":"2024"},{"key":"10.1016\/j.jss.2026.112851_bib0039","unstructured":"Meta, 2024. Meta LLaMA 3.1. Accessed: 2024-09-17. https:\/\/ai.meta.com\/blog\/meta-llama-3-1\/."},{"key":"10.1016\/j.jss.2026.112851_bib0040","unstructured":"OpenAI,. API Documentation, how should i set the temperature parameter?https:\/\/platform.openai.com\/docs\/guides\/text-generation\/how-should-i-set-the-temperature-parameter. Accessed: 2023-11-17."},{"key":"10.1016\/j.jss.2026.112851_bib0041","unstructured":"OpenAI, 2022. ChatGPT: Optimizing Language Models for Dialogue. https:\/\/platform.openai.com\/docs\/models#gpt-3-5-turbo."},{"key":"10.1016\/j.jss.2026.112851_bib0042","unstructured":"OpenAI, 2023. Gpt-4 technical report. arXiv: 2303.08774."},{"key":"10.1016\/j.jss.2026.112851_bib0043","series-title":"2014 IEEE International Conference on Software Maintenance and Evolution","article-title":"How developers\u2019 collaborations identified from different sources tell us about code changes","author":"Panichella","year":"2014"},{"key":"10.1016\/j.jss.2026.112851_bib0044","unstructured":"Radford, A., Narasimhan, K., Salimans, T., Sutskever, I., et al., 2018. Improving language understanding by generative pre-traininghttps:\/\/cdn.openai.com\/research-covers\/language-unsupervised\/language_understanding_paper.pdf."},{"issue":"1","key":"10.1016\/j.jss.2026.112851_bib0045","article-title":"Exploring the limits of transfer learning with a unified text-to-text transformer","volume":"21","author":"Raffel","year":"2020","journal-title":"J. Mach. Learn. Res."},{"key":"10.1016\/j.jss.2026.112851_bib0046","series-title":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)","first-page":"3982","article-title":"Sentence-BERT: sentence embeddings using siamese BERT-networks","author":"Reimers","year":"2019"},{"key":"10.1016\/j.jss.2026.112851_bib0047","unstructured":"Sanh, V., Debut, L., Chaumond, J., Wolf, T., 2020. DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter. arXiv: 1910.01108."},{"issue":"1","key":"10.1016\/j.jss.2026.112851_bib0048","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/505282.505283","article-title":"Machine learning in automated text categorization","volume":"34","author":"Sebastiani","year":"2002","journal-title":"ACM Comput. Surv."},{"key":"10.1016\/j.jss.2026.112851_bib0049","series-title":"2023 IEEE\/ACM International Workshop on Automated Program Repair (APR)","first-page":"23","article-title":"An analysis of the automatic bug fixing performance of chatGPT","author":"Sobania","year":"2023"},{"key":"10.1016\/j.jss.2026.112851_bib0050","unstructured":"G. Team, 2024. Gemini: a family of highly capable multimodal models. 2312.11805. arXiv: 2312.11805."},{"key":"10.1016\/j.jss.2026.112851_bib0051","unstructured":"Touvron, H., Martin, L., Stone, K., Albert, P., Almahairi, A., Babaei, Y., Bashlykov, N., Batra, S., Bhargava, P., Bhosale, S., Bikel, D., Blecher, L., Ferrer, C. C., Chen, M., Cucurull, G., Esiobu, D., Fernandes, J., Fu, J., Fu, W., Fuller, B., Gao, C., Goswami, V., Goyal, N., Hartshorn, A., Hosseini, S., Hou, R., Inan, H., Kardas, M., Kerkez, V., Khabsa, M., Kloumann, I., Korenev, A., Koura, P. S., Lachaux, M.-A., Lavril, T., Lee, J., Liskovich, D., Lu, Y., Mao, Y., Martinet, X., Mihaylov, T., Mishra, P., Molybog, I., Nie, Y., Poulton, A., Reizenstein, J., Rungta, R., Saladi, K., Schelten, A., Silva, R., Smith, E. M., Subramanian, R., Tan, X. E., Tang, B., Taylor, R., Williams, A., Kuan, J. X., Xu, P., Yan, Z., Zarov, I., Zhang, Y., Fan, A., Kambadur, M., Narang, S., Rodriguez, A., Stojnic, R., Edunov, S., Scialom, T., 2023. Llama 2: open foundation and fine-tuned chat models. arXiv: 2307.09288."},{"key":"10.1016\/j.jss.2026.112851_bib0052","unstructured":"Tunstall, L., Beeching, E., Lambert, N., Rajani, N., Rasul, K., Belkada, Y., Huang, S., von Werra, L., Fourrier, C., Habib, N., Sarrazin, N., Sanseviero, O., Rush, A. M., Wolf, T., 2023. Zephyr: direct distillation of LM alignment. arXiv: 2310.16944."},{"key":"10.1016\/j.jss.2026.112851_bib0053","unstructured":"Tunstall, L., Reimers, N., Jo, U. E. S., Bates, L., Korat, D., Wasserblat, M., Pereg, O., 2022. Efficient Few-Shot Learning Without Prompts. 10.48550\/arXiv.2209.11055."},{"key":"10.1016\/j.jss.2026.112851_bib0054","series-title":"Advances in Neural Information Processing Systems","article-title":"Attention is all you need","author":"Vaswani","year":"2017"},{"key":"10.1016\/j.jss.2026.112851_bib0055","article-title":"Understanding interobserver agreement: the kappa statistic","author":"Viera","year":"2005","journal-title":"Fam. Med."},{"issue":"4","key":"10.1016\/j.jss.2026.112851_bib0056","doi-asserted-by":"crossref","first-page":"911","DOI":"10.1109\/TSE.2024.3368208","article-title":"Software testing with large language models: survey, landscape, and vision","volume":"50","author":"Wang","year":"2024","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112851_bib0057","doi-asserted-by":"crossref","DOI":"10.1016\/j.knosys.2021.107476","article-title":"How well do pre-trained contextual language representations recommend labels for github issues?","author":"Wang","year":"2021","journal-title":"Knowl. Based Syst."},{"key":"10.1016\/j.jss.2026.112851_bib0058","unstructured":"Wei, J., Tay, Y., Bommasani, R., Raffel, C., Zoph, B., Borgeaud, S., Yogatama, D., Bosma, M., Zhou, D., Metzler, D., Chi, E. H., Hashimoto, T., Vinyals, O., Liang, P., Dean, J., Fedus, W., 2022a. Emergent abilities of large language models. arXiv: 2206.07682."},{"key":"10.1016\/j.jss.2026.112851_bib0059","series-title":"Advances in Neural Information Processing Systems","first-page":"24824","article-title":"Chain-of-thought prompting elicits reasoning in large language models","author":"Wei","year":"2022"},{"key":"10.1016\/j.jss.2026.112851_bib0060","series-title":"Proceedings of the 6th ACM SIGPLAN International Symposium on Machine Programming","first-page":"1-10","article-title":"A systematic evaluation of large language models of code","author":"Xu","year":"2022"},{"issue":"FSE","key":"10.1016\/j.jss.2026.112851_bib0061","doi-asserted-by":"crossref","DOI":"10.1145\/3660778","article-title":"Exploring and unleashing the power of large language models in automated code translation","volume":"1","author":"Yang","year":"2024","journal-title":"Proc. ACM Softw. Eng."},{"key":"10.1016\/j.jss.2026.112851_bib0062","series-title":"Proceedings of the 18th International Conference on Predictive Models and Data Analytics in Software Engineering","first-page":"62-71","article-title":"Assessing the quality of github copilot\u2019s code generation","author":"Yetistiren","year":"2022"},{"key":"10.1016\/j.jss.2026.112851_bib0063","series-title":"Proceedings of the IEEE\/ACM 46th International Conference on Software Engineering","article-title":"Codereval: a benchmark of pragmatic code generation with generative pre-trained models","author":"Yu","year":"2024"},{"key":"10.1016\/j.jss.2026.112851_bib0064","unstructured":"Yuan, Z., Liu, J., Zi, Q., Liu, M., Peng, X., Lou, Y., 2023. Evaluating instruction-tuned large language models on code comprehension and generation. arXiv: 2308.01240."},{"key":"10.1016\/j.jss.2026.112851_bib0065","unstructured":"Zhao, W. X., Zhou, K., Li, J., Tang, T., Wang, X., Hou, Y., Min, Y., Zhang, B., Zhang, J., Dong, Z., Du, Y., Yang, C., Chen, Y., Chen, Z., Jiang, J., Ren, R., Li, Y., Tang, X., Liu, Z., Liu, P., Nie, J.-Y., Wen, J.-R., 2023. A survey of large language models. arXiv: 2303.18223."},{"issue":"3","key":"10.1016\/j.jss.2026.112851_bib0066","doi-asserted-by":"crossref","first-page":"150","DOI":"10.1002\/smr.1770","article-title":"Combining text mining and data mining for bug report classification","volume":"28","author":"Zhou","year":"2016","journal-title":"J. Softw. Evol. Process"},{"key":"10.1016\/j.jss.2026.112851_bib0067","series-title":"Proceedings of the 20th Chinese National Conference on Computational Linguistics","first-page":"1218","article-title":"A robustly optimized BERT pre-training approach with post-training","author":"Zhuang","year":"2021"}],"container-title":["Journal of Systems and Software"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0164121226000853?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0164121226000853?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,6,11]],"date-time":"2026-06-11T01:10:52Z","timestamp":1781140252000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0164121226000853"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,7]]},"references-count":67,"alternative-id":["S0164121226000853"],"URL":"https:\/\/doi.org\/10.1016\/j.jss.2026.112851","relation":{},"ISSN":["0164-1212"],"issn-type":[{"value":"0164-1212","type":"print"}],"subject":[],"published":{"date-parts":[[2026,7]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Issue classification with LLMs: An empirical study of the NASA flight software systems","name":"articletitle","label":"Article Title"},{"value":"Journal of Systems and Software","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.jss.2026.112851","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 Elsevier Inc. All rights are reserved, including those for text and data mining, AI training, and similar technologies.","name":"copyright","label":"Copyright"}],"article-number":"112851"}}