{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,19]],"date-time":"2026-01-19T23:48:23Z","timestamp":1768866503934,"version":"3.49.0"},"reference-count":45,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2025,9,23]],"date-time":"2025-09-23T00:00:00Z","timestamp":1758585600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,9,23]],"date-time":"2025-09-23T00:00:00Z","timestamp":1758585600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62372021"],"award-info":[{"award-number":["62372021"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100012226","name":"Fundamental Research Funds for the Central Universities","doi-asserted-by":"publisher","award":["2023JBZY036"],"award-info":[{"award-number":["2023JBZY036"]}],"id":[{"id":"10.13039\/501100012226","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003787","name":"Hebei Natural Science Foundation","doi-asserted-by":"crossref","award":["F2023105005"],"award-info":[{"award-number":["F2023105005"]}],"id":[{"id":"10.13039\/501100003787","id-type":"DOI","asserted-by":"crossref"}]},{"name":"Central Funds Guiding the Local Science and Technology Development","award":["236Z0806G"],"award-info":[{"award-number":["236Z0806G"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Autom Softw Eng"],"published-print":{"date-parts":[[2026,5]]},"DOI":"10.1007\/s10515-025-00555-z","type":"journal-article","created":{"date-parts":[[2025,9,23]],"date-time":"2025-09-23T01:17:23Z","timestamp":1758590243000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Investigating the bugs in reinforcement learning programs: Insights from Stack Overflow and GitHub"],"prefix":"10.1007","volume":"33","author":[{"given":"Jiayin","family":"Song","sequence":"first","affiliation":[]},{"given":"Yike","family":"Li","sequence":"additional","affiliation":[]},{"given":"Yunzhe","family":"Tian","sequence":"additional","affiliation":[]},{"given":"Haoxuan","family":"Ma","sequence":"additional","affiliation":[]},{"given":"Honglei","family":"Li","sequence":"additional","affiliation":[]},{"given":"Jie","family":"Zuo","sequence":"additional","affiliation":[]},{"given":"Jiqiang","family":"Liu","sequence":"additional","affiliation":[]},{"given":"Wenjia","family":"Niu","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,9,23]]},"reference":[{"key":"555_CR1","unstructured":"Akkaya, I., Andrychowicz, M., Chociej, M., Litwin, M., McGrew, B., Petron, A., Paino, A., Plappert, M., Powell, G., Ribas, R., et\u00a0al.: Solving rubik\u2019s cube with a robot hand. arXiv preprint arXiv:1910.07113. (2019)"},{"key":"555_CR2","doi-asserted-by":"publisher","unstructured":"Alharthi, H., Outioua, D., Baysal, O.: Predicting questions\u2019 scores on stack overflow. In: 2016 IEEE\/ACM 3rd international workshop on crowdsourcing in software engineering (CSI-SE), pp. 1\u20137 (2016). https:\/\/doi.org\/10.1145\/2897659.2897661","DOI":"10.1145\/2897659.2897661"},{"key":"555_CR3","doi-asserted-by":"publisher","unstructured":"Borges, H., Hora, A., Valente, M.T.: Understanding the factors that impact the popularity of github repositories. In: 2016 IEEE international conference on software maintenance and evolution (ICSME), pp. 334\u2013344 (2016). https:\/\/doi.org\/10.1109\/ICSME.2016.31","DOI":"10.1109\/ICSME.2016.31"},{"key":"555_CR4","doi-asserted-by":"crossref","unstructured":"Bouchoucha, R., Yahmed, A.H., Patil, D., Rajendran, J., Nikanjam, A., Chandar, S., Khomh, F.: Toward debugging deep reinforcement learning programs with rlexplorer. arXiv preprint arXiv:2410.04322. (2024)","DOI":"10.1109\/ICSME58944.2024.00019"},{"issue":"4","key":"555_CR5","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3394112","volume":"29","author":"J Chen","year":"2020","unstructured":"Chen, J., Wu, Z., Wang, Z., You, H., Zhang, L., Yan, M.: Practical accuracy estimation for efficient deep neural network testing. ACM Trans. Softw. Eng. Methodol. (TOSEM) 29(4), 1\u201335 (2020)","journal-title":"ACM Trans. Softw. Eng. Methodol. (TOSEM)"},{"key":"555_CR6","first-page":"1","volume":"32","author":"J Chen","year":"2022","unstructured":"Chen, J., Liang, Y., Shen, Q., Jiang, J., Li, S.: Toward understanding deep learning framework bugs. ACM Trans. Softw. Eng. Methodol. 32, 1\u201331 (2022)","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"issue":"6","key":"555_CR7","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3587155","volume":"32","author":"J Chen","year":"2023","unstructured":"Chen, J., Liang, Y., Shen, Q., Jiang, J., Li, S.: Toward understanding deep learning framework bugs. ACM Trans. Softw. Eng. Methodol. 32(6), 1\u201331 (2023)","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"555_CR8","doi-asserted-by":"crossref","unstructured":"Du, X., Xiao, G., Sui, Y.: Fault triggers in the tensorflow framework: An experience report. In: 2020 IEEE 31st international symposium on software reliability engineering (ISSRE), pp. 1\u201312 (2020). IEEE","DOI":"10.1109\/ISSRE5003.2020.00010"},{"issue":"4","key":"555_CR9","doi-asserted-by":"publisher","first-page":"2696","DOI":"10.1109\/TDSC.2022.3152239","volume":"20","author":"X Du","year":"2022","unstructured":"Du, X., Sui, Y., Liu, Z., Ai, J.: An empirical study of fault triggers in deep learning frameworks. IEEE Trans. Dependable Secure Comput. 20(4), 2696\u20132712 (2022)","journal-title":"IEEE Trans. Dependable Secure Comput."},{"key":"555_CR10","doi-asserted-by":"publisher","first-page":"2696","DOI":"10.1109\/TDSC.2022.3152239","volume":"20","author":"X Du","year":"2023","unstructured":"Du, X., Sui, Y., Liu, Z., Ai, J.: An empirical study of fault triggers in deep learning frameworks. IEEE Trans. Dependable Secure Comput. 20, 2696\u20132712 (2023)","journal-title":"IEEE Trans. Dependable Secure Comput."},{"key":"555_CR11","unstructured":"Fischer, T.G.: Reinforcement learning in financial markets-a survey. Technical report, FAU discussion papers in economics (2018)"},{"key":"555_CR12","doi-asserted-by":"crossref","unstructured":"Garcia, J., Feng, Y., Shen, J., Almanee, S., Xia, Y., Chen, Alfred, Q.: A comprehensive study of autonomous vehicle bugs. In: Proceedings of the ACM\/IEEE 42nd international conference on software engineering, pp. 385\u2013396 (2020)","DOI":"10.1145\/3377811.3380397"},{"key":"555_CR13","doi-asserted-by":"crossref","unstructured":"Haque, M.U., Iwaya, L.H., Babar, M.A.: Challenges in docker development: A large-scale study using stack overflow. In: Proceedings of the 14th ACM\/IEEE international symposium on empirical software engineering and measurement (ESEM), pp. 1\u201311 (2020)","DOI":"10.1145\/3382494.3410693"},{"key":"555_CR14","doi-asserted-by":"crossref","unstructured":"Humbatova, N., Jahangirova, G., Bavota, G., Riccio, V., Stocco, A., Tonella, P.: Taxonomy of real faults in deep learning systems. In: Proceedings of the ACM\/IEEE 42nd international conference on software engineering, pp. 1110\u20131121 (2020)","DOI":"10.1145\/3377811.3380395"},{"key":"555_CR15","doi-asserted-by":"crossref","unstructured":"Islam, M.J., Nguyen, G., Pan, R., Rajan, H.: A comprehensive study on deep learning bug characteristics. In: Proceedings of the 2019 27th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, pp. 510\u2013520 (2019)","DOI":"10.1145\/3338906.3338955"},{"key":"555_CR16","doi-asserted-by":"crossref","unstructured":"Jia, L., Zhong, H., Wang, X., Huang, L., Lu, X.: An empirical study on bugs inside tensorflow. In: Database systems for advanced applications: 25th international conference, DASFAA 2020, Jeju, South Korea, September 24\u201327, 2020, Proceedings, Part I 25, pp. 604\u2013620 (2020). Springer","DOI":"10.1007\/978-3-030-59410-7_40"},{"key":"555_CR17","volume-title":"Program Evolution: Processes of Software Change","author":"MM Lehman","year":"1985","unstructured":"Lehman, M.M., Belady, L.A.: Program Evolution: Processes of Software Change. Academic Press Professional Inc, USA (1985)"},{"key":"555_CR18","doi-asserted-by":"publisher","unstructured":"Liao, Z., Zhao, B., Liu, S., Jin, H., He, D., Yang, L., Zhang, Y., Wu, J.: A prediction model of the project life-span in open source software ecosystem. Mob. Netw. Appl. 24(4), 1382\u20131391 (2019). https:\/\/doi.org\/10.1007\/s11036-018-0993-3","DOI":"10.1007\/s11036-018-0993-3"},{"key":"555_CR19","doi-asserted-by":"crossref","unstructured":"Long, G., Chen, T.: On reporting performance and accuracy bugs for deep learning frameworks: An exploratory study from github. In: Proceedings of the 26th international conference on evaluation and assessment in software engineering, pp. 90\u201399 (2022)","DOI":"10.1145\/3530019.3530029"},{"key":"555_CR20","doi-asserted-by":"crossref","unstructured":"Mann, H.B.: Nonparametric tests against trend. Econometrica J. Econ. Soc. 245\u2013259 (1945)","DOI":"10.2307\/1907187"},{"key":"555_CR21","doi-asserted-by":"crossref","unstructured":"Meng, F., Wang, X., Wang, J., Wang, P.: Automatic classification of bug reports based on multiple text information and reports\u2019 intention. In: International symposium on theoretical aspects of software engineering, pp. 131\u2013147 (2022). Springer","DOI":"10.1007\/978-3-031-10363-6_9"},{"key":"555_CR22","unstructured":"Mnih, V.: Playing atari with deep reinforcement learning. arXiv preprint arXiv:1312.5602. (2013)"},{"issue":"4","key":"555_CR23","doi-asserted-by":"publisher","first-page":"95","DOI":"10.1007\/s10664-024-10500-5","volume":"29","author":"MM Morovati","year":"2024","unstructured":"Morovati, M.M., Tambon, F., Taraghi, M., Nikanjam, A., Khomh, F.: Common challenges of deep reinforcement learning applications development: an empirical study. Empir. Softw. Eng. 29(4), 95 (2024)","journal-title":"Empir. Softw. Eng."},{"issue":"1","key":"555_CR24","doi-asserted-by":"publisher","first-page":"8","DOI":"10.1007\/s10515-021-00313-x","volume":"29","author":"A Nikanjam","year":"2022","unstructured":"Nikanjam, A., Morovati, M.M., Khomh, F., Ben Braiek, H.: Faults in deep reinforcement learning programs: A taxonomy and a detection approach. Autom. Softw Eng. 29(1), 8 (2022)","journal-title":"Autom. Softw Eng."},{"key":"555_CR25","doi-asserted-by":"crossref","unstructured":"Pham, H.V., Lutellier, T., Qi, W., Tan, L.: Cradle: cross-backend validation to detect and localize bugs in deep learning libraries. In: 2019 IEEE\/ACM 41st international conference on software engineering (ICSE), pp. 1027\u20131038 (2019). IEEE","DOI":"10.1109\/ICSE.2019.00107"},{"key":"555_CR26","doi-asserted-by":"crossref","unstructured":"Sallab, A.E., Abdou, M., Perot, E., Yogamani, S.: Deep reinforcement learning framework for autonomous driving. arXiv preprint arXiv:1704.02532. (2017)","DOI":"10.2352\/ISSN.2470-1173.2017.19.AVM-023"},{"issue":"7839","key":"555_CR27","doi-asserted-by":"publisher","first-page":"604","DOI":"10.1038\/s41586-020-03051-4","volume":"588","author":"J Schrittwieser","year":"2020","unstructured":"Schrittwieser, J., Antonoglou, I., Hubert, T., Simonyan, K., Sifre, L., Schmitt, S., Guez, A., Lockhart, E., Hassabis, D., Graepel, T., et al.: Mastering atari, go, chess and shogi by planning with a learned model. Nat. 588(7839), 604\u2013609 (2020)","journal-title":"Nat."},{"key":"555_CR28","doi-asserted-by":"crossref","unstructured":"Shen, Q., Ma, H., Chen, J., Tian, Y., Cheung, S.-C., Chen, X.: A comprehensive study of deep learning compiler bugs. In: Proceedings of the 29th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, pp. 968\u2013980 (2021)","DOI":"10.1145\/3468264.3468591"},{"key":"555_CR29","unstructured":"Sutton, R.S.: Reinforcement learning: An introduction. A Bradford Book. (2018)"},{"key":"555_CR30","doi-asserted-by":"crossref","unstructured":"Tambon, F., Majdinasab, V., Nikanjam, A., Khomh, F., Antoniol, G.: Mutation testing of deep reinforcement learning based on real faults. In: 2023 IEEE conference on software testing, verification and validation (ICST), pp. 188\u2013198 (2023). IEEE","DOI":"10.1109\/ICST57152.2023.00026"},{"issue":"1","key":"555_CR31","doi-asserted-by":"publisher","first-page":"10","DOI":"10.1007\/s10664-023-10389-6","volume":"29","author":"F Tambon","year":"2024","unstructured":"Tambon, F., Nikanjam, A., An, L., Khomh, F., Antoniol, G.: Silent bugs in deep learning frameworks: an empirical study of keras and tensorflow. Empir. Softw. Eng. 29(1), 10 (2024)","journal-title":"Empir. Softw. Eng."},{"key":"555_CR32","doi-asserted-by":"publisher","unstructured":"Tan, X., Zhou, M., Sun, Z.: A first look at good first issues on github. In: Proceedings of the 28th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering. ESEC\/FSE 2020, pp. 398\u2013409. Association for Computing Machinery, New York, NY, USA (2020). https:\/\/doi.org\/10.1145\/3368089.3409746","DOI":"10.1145\/3368089.3409746"},{"key":"555_CR33","doi-asserted-by":"crossref","unstructured":"Todorov, E., Erez, T., Tassa, Y.: Mujoco: A physics engine for model-based control. In: 2012 IEEE\/RSJ international conference on intelligent robots and systems, pp. 5026\u20135033 (2012). IEEE","DOI":"10.1109\/IROS.2012.6386109"},{"key":"555_CR34","doi-asserted-by":"crossref","unstructured":"Vieira, S.M., Kaymak, U., Sousa, J.M.C.: Cohen\u2019s kappa coefficient as a performance measure for feature selection. Int. Conf. Fuzzy Syst. 1\u20138 (2010)","DOI":"10.1109\/FUZZY.2010.5584447"},{"key":"555_CR35","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2024.111963","volume":"210","author":"X Wan","year":"2024","unstructured":"Wan, X., Li, T., Lin, W., Cai, Y., Zheng, Z.: Coverage-guided fuzzing for deep reinforcement learning systems. J. Syst. Softw. 210, 111963 (2024)","journal-title":"J. Syst. Softw."},{"key":"555_CR36","doi-asserted-by":"crossref","unstructured":"Wang, Z., Yan, M., Chen, J., Liu, S., Zhang, D.: Deep learning library testing via effective model generation. In: Proceedings of the 28th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, pp. 788\u2013799 (2020)","DOI":"10.1145\/3368089.3409761"},{"key":"555_CR37","doi-asserted-by":"crossref","unstructured":"Wang, Z., You, H., Chen, J., Zhang, Y., Dong, X., Zhang, W.: Prioritizing test inputs for deep neural networks via mutation analysis. In: 2021 IEEE\/ACM 43rd international conference on software engineering (ICSE), pp. 397\u2013409 (2021). IEEE","DOI":"10.1109\/ICSE43902.2021.00046"},{"key":"555_CR38","doi-asserted-by":"crossref","unstructured":"Yahmed, A.H., Abbassi, A.A., Nikanjam, A., Li, H., Khomh, F.: Deploying deep reinforcement learning systems: A taxonomy of challenges. In: 2023 IEEE international conference on software maintenance and evolution (ICSME), pp. 26\u201338 (2023). IEEE","DOI":"10.1109\/ICSME58846.2023.00015"},{"key":"555_CR39","doi-asserted-by":"crossref","unstructured":"Yan, M., Chen, J., Zhang, X., Tan, L., Wang, G., Wang, Z.: Exposing numerical bugs in deep learning via gradient back-propagation. In: Proceedings of the 29th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, pp. 627\u2013638 (2021)","DOI":"10.1145\/3468264.3468612"},{"key":"555_CR40","doi-asserted-by":"crossref","unstructured":"Zahedi, M., Rajapakse, R.N., Babar, M.A.: Mining questions asked about continuous software engineering: A case study of stack overflow. In: Proceedings of the 24th international conference on evaluation and assessment in software engineering, pp. 41\u201350 (2020)","DOI":"10.1145\/3383219.3383224"},{"key":"555_CR41","doi-asserted-by":"publisher","unstructured":"Zhang, Y., Chen, Y., Cheung, S.-C., Xiong, Y., Zhang, L.: An empirical study on TensorFlow program bugs. In: Proceedings of the 27th ACM SIGSOFT International Symposium on Software Testing and Analysis, pp. 129\u2013140. ACM, Amsterdam Netherlands (2018). https:\/\/doi.org\/10.1145\/3213846.3213866. Accessed 16 Aug 2023","DOI":"10.1145\/3213846.3213866"},{"key":"555_CR42","doi-asserted-by":"crossref","unstructured":"Zhang, T., Gao, C., Ma, L., Lyu, M., Kim, M.: An empirical study of common challenges in developing deep learning applications. In: 2019 IEEE 30th international symposium on software reliability engineering (ISSRE), pp. 104\u2013115 (2019). IEEE","DOI":"10.1109\/ISSRE.2019.00020"},{"key":"555_CR43","doi-asserted-by":"crossref","unstructured":"Zhang, Y., Ren, L., Chen, L., Xiong, Y., Cheung, S.-C., Xie, T.: Detecting numerical bugs in neural network architectures. In: Proceedings of the 28th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, pp. 826\u2013837 (2020)","DOI":"10.1145\/3368089.3409720"},{"key":"555_CR44","unstructured":"Zhang, Z., Zhang, A., Li, M., Smola, A.J.: Automatic chain of thought prompting in large language models. ArXiv. abs\/2210.03493 (2022)"},{"key":"555_CR45","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2023.3269804","volume-title":"A search-based testing approach for deep reinforcement learning agents","author":"A Zolfagharian","year":"2023","unstructured":"Zolfagharian, A., Abdellatif, M., Briand, L.C., Bagherzadeh, M., Ramesh, S.: A search-based testing approach for deep reinforcement learning agents. IEEE Trans. Softw, Eng (2023)"}],"container-title":["Automated Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10515-025-00555-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10515-025-00555-z","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10515-025-00555-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,19]],"date-time":"2026-01-19T11:16:08Z","timestamp":1768821368000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10515-025-00555-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,9,23]]},"references-count":45,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2026,5]]}},"alternative-id":["555"],"URL":"https:\/\/doi.org\/10.1007\/s10515-025-00555-z","relation":{},"ISSN":["0928-8910","1573-7535"],"issn-type":[{"value":"0928-8910","type":"print"},{"value":"1573-7535","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,9,23]]},"assertion":[{"value":"7 February 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 August 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 September 2025","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflicts of Interest"}}],"article-number":"9"}}