{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,21]],"date-time":"2025-10-21T15:51:09Z","timestamp":1761061869750},"reference-count":63,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2023,8,24]],"date-time":"2023-08-24T00:00:00Z","timestamp":1692835200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,8,24]],"date-time":"2023-08-24T00:00:00Z","timestamp":1692835200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2023,9]]},"DOI":"10.1007\/s10664-023-10363-2","type":"journal-article","created":{"date-parts":[[2023,8,24]],"date-time":"2023-08-24T06:02:10Z","timestamp":1692856930000},"update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":11,"title":["A comparison of reinforcement learning frameworks for software testing tasks"],"prefix":"10.1007","volume":"28","author":[{"given":"Paulina Stevia","family":"Nouwou Mindom","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Amin","family":"Nikanjam","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Foutse","family":"Khomh","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,8,24]]},"reference":[{"key":"10363_CR1","unstructured":"Cartpole (2016). https:\/\/gym.openai.com\/envs\/CartPole-v0\/"},{"key":"10363_CR2","unstructured":"Mspacman (2018). https:\/\/gym.openai.com\/envs\/MsPacman-v0\/"},{"key":"10363_CR3","unstructured":"Replication package (2022). https:\/\/github.com\/npaulinastevia\/DRL_se"},{"key":"10363_CR4","unstructured":"Abadi M, Agarwal A, Barham P, Brevdo E, Chen Z, Citro C, Corrado GS, Davis A, Dean J, Devin M, Ghemawat S, Goodfellow I, Harp A, Irving G, Isard M, Jozefowicz R, Jia Y, Kaiser L, Kudlur M, Levenberg J, Man\u00e9 D, Schuster M, Monga R, Moore S, Murray D, Olah C, Shlens J, Steiner B, Sutskever I, Talwar K, Tucker P, Vanhoucke V, Vasudevan V, Vi\u00e9gas F, Vinyals O, Warden P, Wattenberg M, Wicke M, Yu Y, Zheng X (2015) Tensorflow, large-scale machine learning on heterogeneous systems. 10.5281\/zenodo.4724125"},{"key":"10363_CR5","doi-asserted-by":"crossref","unstructured":"Adamo D, Khan MK, Koppula S, Bryce R (2018) Reinforcement learning for android gui testing. In: Proceedings of the 9th ACM SIGSOFT International Workshop on Automating TEST Case Design, Selection, and Evaluation, pp 2\u20138","DOI":"10.1145\/3278186.3278187"},{"key":"10363_CR6","doi-asserted-by":"crossref","unstructured":"Alshahwan N, Gao X, Harman M, Jia Y, Mao K, Mols A, Tei T, Zorin I (2018) Deploying search based software engineering with sapienz at facebook. In: International Symposium on Search Based Software Engineering, Springer, pp 3\u201345","DOI":"10.1007\/978-3-319-99241-9_1"},{"issue":"3","key":"10363_CR7","doi-asserted-by":"publisher","first-page":"219","DOI":"10.1002\/stvr.1486","volume":"24","author":"A Arcuri","year":"2014","unstructured":"Arcuri A, Briand L (2014) A hitchhiker\u2019s guide to statistical tests for assessing randomized algorithms in software engineering. Softw Test Verification Reliab 24(3):219\u2013250","journal-title":"Softw Test Verification Reliab"},{"key":"10363_CR8","doi-asserted-by":"crossref","unstructured":"Bagherzadeh M, Kahani N, Briand L (2021) Reinforcement learning for test case prioritization. IEEE Transactions on Software Engineering","DOI":"10.1109\/TSE.2021.3070549"},{"issue":"12","key":"10363_CR9","doi-asserted-by":"publisher","first-page":"1209","DOI":"10.1007\/s00607-015-0455-8","volume":"97","author":"F Bahrpeyma","year":"2015","unstructured":"Bahrpeyma F, Haghighi H, Zakerolhosseini A (2015) An adaptive rl based approach for dynamic resource provisioning in cloud virtualized data centers. Computing 97(12):1209\u20131234","journal-title":"Computing"},{"key":"10363_CR10","doi-asserted-by":"crossref","unstructured":"Bergdahl J, Gordillo C, Tollmar K, Gissl\u00e9n L (2020) Augmenting automated game testing with deep reinforcement learning. In: 2020 IEEE Conference on Games (CoG), IEEE, pp 600\u2013603","DOI":"10.1109\/CoG47356.2020.9231552"},{"key":"10363_CR11","doi-asserted-by":"crossref","unstructured":"Bertolino A, Guerriero A, Miranda B, Pietrantuono R, Russo S (2020) Learning-to-rank vs ranking-to-learn: strategies for regression testing in continuous integration. In: Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering, pp 1\u201312","DOI":"10.1145\/3377811.3380369"},{"key":"10363_CR12","doi-asserted-by":"crossref","unstructured":"B\u00f6ttinger K, Godefroid P, Singh R (2018) Deep reinforcement fuzzing. In: 2018 IEEE Security and Privacy Workshops (SPW), IEEE, pp 116\u2013122","DOI":"10.1109\/SPW.2018.00026"},{"key":"10363_CR13","unstructured":"Brockman G, Cheung V, Pettersson L, Schneider J, Schulman J, Tang J, Zaremba W (2016a) Openai gym. arXiv:1606.01540"},{"key":"10363_CR14","unstructured":"Brockman G, Cheung V, Pettersson L, Schneider J, Schulman J, Tang J, Zaremba W (2016b) Openai gym. arXiv:1606.01540"},{"key":"10363_CR15","unstructured":"Castro PS, Moitra S, Gelada C, Kumar S, Bellemare MG (2018) Dopamine: A research framework for deep reinforcement learning. arXiv preprint arXiv:1812.06110"},{"key":"10363_CR16","doi-asserted-by":"crossref","unstructured":"Chen J, Ma H, Zhang L (2020) Enhanced compiler bug isolation via memoized search. In: Proceedings of the 35th IEEE\/ACM International Conference on Automated Software Engineering, pp 78\u201389","DOI":"10.1145\/3324884.3416570"},{"key":"10363_CR17","unstructured":"Dai H, Li Y, Wang C, Singh R, Huang PS, Kohli P (2019) Learning transferable graph exploration. Advances in Neural Information Processing Systems 32"},{"key":"10363_CR18","unstructured":"Dhariwal P, Hesse C, Klimov O, Nichol A, Plappert M, Radford A, Schulman J, Sidor S, Wu Y, Zhokhov P (2017) Openai baselines. https:\/\/github.com\/openai\/baselines"},{"key":"10363_CR19","unstructured":"Drozd W, Wagner MD (2018) Fuzzergym: A competitive framework for fuzzing and learning. arXiv preprint arXiv:1807.07490"},{"key":"10363_CR20","unstructured":"Dulac-Arnold G, Mankowitz D, Hester T (2019) Challenges of real-world reinforcement learning. arXiv preprint arXiv:1904.12901"},{"key":"10363_CR21","unstructured":"Fortunato M, Azar MG, Piot B, Menick J, Osband I, Graves A, Mnih V, Munos R, Hassabis D, Pietquin O, et\u00a0al. (2017) Noisy networks for exploration. arXiv preprint arXiv:1706.10295"},{"key":"10363_CR22","doi-asserted-by":"crossref","unstructured":"Fraser G, Arcuri A (2011) Evosuite: automatic test suite generation for object-oriented software. In: Proceedings of the 19th ACM SIGSOFT symposium and the 13th European conference on Foundations of software engineering, pp 416\u2013419","DOI":"10.1145\/2025113.2025179"},{"key":"10363_CR23","unstructured":"Fujimoto S, Hoof H, Meger D (2018) Addressing function approximation error in actor-critic methods. In: International Conference on Machine Learning, PMLR, pp 1587\u20131596"},{"issue":"2","key":"10363_CR24","first-page":"113","volume":"1","author":"PA Games","year":"1976","unstructured":"Games PA, Howell JF (1976) Pairwise multiple comparison procedures with unequal n\u2019s and\/or variances: a monte carlo study. J Educ Stat 1(2):113\u2013125","journal-title":"J Educ Stat"},{"key":"10363_CR25","unstructured":"Gu S, Lillicrap T, Sutskever I, Levine S (2016) Continuous deep q-learning with model-based acceleration. In: International conference on machine learning, PMLR, pp 2829\u20132838"},{"key":"10363_CR26","unstructured":"Haarnoja T, Zhou A, Abbeel P, Levine S (2018) Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor. In: International conference on machine learning, PMLR, pp 1861\u20131870"},{"key":"10363_CR27","first-page":"970","volume-title":"Random testing, encyclopedia of software engineering","author":"R Hamlet","year":"1994","unstructured":"Hamlet R, Maciniak J (1994) Random testing, encyclopedia of software engineering. Wiley, New York, pp 970\u2013978"},{"key":"10363_CR28","first-page":"1","volume-title":"2015 IEEE 8th International Conference on Software Testing","author":"M Harman","year":"2015","unstructured":"Harman M, Jia Y, Zhang Y (2015) Achievements, open problems and challenges for search based software testing. 2015 IEEE 8th International Conference on Software Testing. Verification and Validation (ICST), IEEE, pp 1\u201312"},{"key":"10363_CR29","unstructured":"Hill A, Raffin A, Ernestus M, Gleave A, Kanervisto A, Traore R, Dhariwal P, Hesse C, Klimov O, Nichol A, Plappert M, Radford A, Schulman J, Sidor S, Wu Y (2018) Stable baselines. https:\/\/github.com\/hill-a\/stable-baselines"},{"key":"10363_CR30","unstructured":"Hill A, Raffin A, Ernestus M, Gleave A, Kanervisto A, Traore R, Dhariwal P, Hesse C, Klimov O, Nichol A et\u00a0al (2019) Stable baselines. 2018. https:\/\/github.com\/hill-a\/stable-baselines"},{"key":"10363_CR31","doi-asserted-by":"crossref","unstructured":"Kim J, Kwon M, Yoo S (2018) Generating test input with deep reinforcement learning. In: 2018 IEEE\/ACM 11th International Workshop on Search-Based Software Testing (SBST), IEEE, pp 51\u201358","DOI":"10.1145\/3194718.3194720"},{"key":"10363_CR32","unstructured":"Kingma DP, Ba J (2014) Adam: A method for stochastic optimization. arXiv preprint arXiv:1412.6980"},{"key":"10363_CR33","unstructured":"Knuth DE (1997) The art of computer programming, vol\u00a03. Pearson Education"},{"key":"10363_CR34","first-page":"105","volume-title":"2018 IEEE 11th International Conference on Software Testing","author":"Y Koroglu","year":"2018","unstructured":"Koroglu Y, Sen A, Muslu O, Mete Y, Ulker C, Tanriverdi T, Donmez Y (2018) Qbe: Qlearning-based exploration of android applications. 2018 IEEE 11th International Conference on Software Testing. Verification and Validation (ICST), IEEE, pp 105\u2013115"},{"key":"10363_CR35","unstructured":"Lillicrap TP, Hunt JJ, Pritzel A, Heess N, Erez T, Tassa Y, Silver D, Wierstra D (2015) Continuous control with deep reinforcement learning. arXiv preprint arXiv:1509.02971"},{"issue":"3","key":"10363_CR36","doi-asserted-by":"publisher","first-page":"234","DOI":"10.1080\/09540091.2015.1031082","volume":"27","author":"K Malialis","year":"2015","unstructured":"Malialis K, Devlin S, Kudenko D (2015) Distributed reinforcement learning for adaptive and robust network intrusion response. Connect Sci 27(3):234\u2013252","journal-title":"Connect Sci"},{"issue":"2","key":"10363_CR37","doi-asserted-by":"publisher","first-page":"361","DOI":"10.1037\/0033-2909.111.2.361","volume":"111","author":"KO McGraw","year":"1992","unstructured":"McGraw KO, Wong SP (1992) A common language effect size statistic. Psychol Bull 111(2):361","journal-title":"Psychol Bull"},{"key":"10363_CR38","unstructured":"Mnih V, Kavukcuoglu K, Silver D, Graves A, Antonoglou I, Wierstra D, Riedmiller M (2013) Playing atari with deep reinforcement learning. arXiv preprint arXiv:1312.5602"},{"key":"10363_CR39","unstructured":"Mnih V, Badia AP, Mirza M, Graves A, Lillicrap T, Harley T, Silver D, Kavukcuoglu K (2016) Asynchronous methods for deep reinforcement learning. In: International conference on machine learning, PMLR, pp 1928\u20131937"},{"key":"10363_CR40","doi-asserted-by":"crossref","unstructured":"Moghadam MH, Saadatmand M, Borg M, Bohlin M, Lisper B (2021) An autonomous performance testing framework using self-adaptive fuzzy reinforcement learning. Softw Qual J 1\u201333","DOI":"10.1007\/s11219-020-09532-z"},{"key":"10363_CR41","unstructured":"Paszke A, Gross S, Massa F, Lerer A, Bradbury J, Chanan G, Killeen T, Lin Z, Gimelshein N, Antiga L, Desmaison A, Kopf A, Yang E, DeVito Z, Raison M, Tejani A, Chilamkurthy S, Steiner B, Fang L, Bai J, Chintala S (2019) Pytorch: An imperative style, high-performance deep learning library. In: Wallach H, Larochelle H, Beygelzimer A, d\u2019Alch\u00e9-Buc F, Fox E, Garnett R (eds) Advances in Neural Information Processing Systems 32, Curran Associates, Inc., pp 8024\u20138035. http:\/\/papers.neurips.cc\/paper\/9015-pytorch-an-imperative-style-high-performance-deep-learning-library.pdf"},{"key":"10363_CR42","unstructured":"Plappert M (2016) keras-rl. https:\/\/github.com\/keras-rl\/keras-rl"},{"key":"10363_CR43","unstructured":"Raffin A, Hill A, Gleave A, Kanervisto A, Ernestus M, Dormann N (2021) Stable-baselines3: Reliable reinforcement learning implementations. Journal of Machine Learning Research 22(268):1\u20138. http:\/\/jmlr.org\/papers\/v22\/20-1364.html"},{"key":"10363_CR44","doi-asserted-by":"crossref","unstructured":"Reichstaller A, Knapp A (2018) Risk-based testing of self-adaptive systems using run-time predictions. In: 2018 IEEE 12th international conference on self-adaptive and self-organizing systems (SASO), IEEE, pp 80\u201389","DOI":"10.1109\/SASO.2018.00019"},{"key":"10363_CR45","doi-asserted-by":"crossref","unstructured":"Romdhana A, Merlo A, Ceccato M, Tonella P (2022) Deep reinforcement learning for black-box testing of android apps. ACM Transactions on Software Engineering and Methodology","DOI":"10.1109\/PerComWorkshops51409.2021.9431072"},{"key":"10363_CR46","doi-asserted-by":"publisher","unstructured":"Santos RES, Magalh\u00e3es CVC, Capretz LF, Correia-Neto JS, da\u00a0Silva FQB, Saher A (2018) Computer games are serious business and so is their quality: Particularities of software testing in game development from the perspective of practitioners. In: Proceedings of the 12th ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement, Association for Computing Machinery, New York, NY, USA, ESEM \u201918. https:\/\/doi.org\/10.1145\/3239235.3268923","DOI":"10.1145\/3239235.3268923"},{"key":"10363_CR47","unstructured":"Schaarschmidt M, Kuhnle A, Ellis B, Fricke K, Gessert F, Yoneki E (2018) Lift: Reinforcement learning in computer systems by learning from demonstrations. arXiv preprint arXiv:1808.07903"},{"key":"10363_CR48","unstructured":"Schulman J, Levine S, Abbeel P, Jordan M, Moritz P (2015) Trust region policy optimization. In: International conference on machine learning, PMLR, pp 1889\u20131897"},{"key":"10363_CR49","unstructured":"Schulman J, Wolski F, Dhariwal P, Radford A, Klimov O (2017) Proximal policy optimization algorithms. arXiv preprint arXiv:1707.06347"},{"key":"10363_CR50","doi-asserted-by":"crossref","unstructured":"Singh L, Sharma DK (2013) An architecture for extracting information from hidden web databases using intelligent agent technology through reinforcement learning. In: 2013 IEEE conference on Information & Communication Technologies, IEEE, pp 292\u2013297","DOI":"10.1109\/CICT.2013.6558108"},{"issue":"2","key":"10363_CR51","doi-asserted-by":"publisher","first-page":"553","DOI":"10.1109\/TCC.2018.2805812","volume":"8","author":"M Soualhia","year":"2020","unstructured":"Soualhia M, Khomh F, Tahar S (2020) A dynamic and failure-aware task scheduling framework for hadoop. IEEE Trans Cloud Comput 8(2):553\u2013569. https:\/\/doi.org\/10.1109\/TCC.2018.2805812","journal-title":"IEEE Trans Cloud Comput"},{"key":"10363_CR52","doi-asserted-by":"crossref","unstructured":"Spieker H, Gotlieb A, Marijan D, Mossige M (2017) Reinforcement learning for automatic test case prioritization and selection in continuous integration. In: Proceedings of the 26th ACM SIGSOFT International Symposium on Software Testing and Analysis, pp 12\u201322","DOI":"10.1145\/3092703.3092709"},{"key":"10363_CR53","unstructured":"Such FP, Madhavan V, Conti E, Lehman J, Stanley KO, Clune J (2017) Deep neuroevolution: Genetic algorithms are a competitive alternative for training deep neural networks for reinforcement learning. arXiv preprint arXiv:1712.06567"},{"key":"10363_CR54","unstructured":"Sutton RS, Barto AG, et\u00a0al. (1998) Introduction to reinforcement learning, vol 135. MIT press Cambridge"},{"key":"10363_CR55","doi-asserted-by":"crossref","unstructured":"Tufano R, Scalabrino S, Pascarella L, Aghajani E, Oliveto R, Bavota G (2022) Using reinforcement learning for load testing of video games. In: Proceedings of the 44th International Conference on Software Engineering, pp 2303\u20132314","DOI":"10.1145\/3510003.3510625"},{"key":"10363_CR56","doi-asserted-by":"crossref","unstructured":"Vuong TAT, Takada S (2018) A reinforcement learning based approach to automated testing of android applications. In: Proceedings of the 9th ACM SIGSOFT International Workshop on Automating TEST Case Design, Selection, and Evaluation, pp 31\u201337","DOI":"10.1145\/3278186.3278191"},{"key":"10363_CR57","unstructured":"Wang Z, Schaul T, Hessel M, Hasselt H, Lanctot M, Freitas N (2016) Dueling network architectures for deep reinforcement learning. In: International conference on machine learning, PMLR, pp 1995\u20132003"},{"issue":"1\u20132","key":"10363_CR58","first-page":"28","volume":"34","author":"BL Welch","year":"1947","unstructured":"Welch BL (1947) The generalization of \u2018student\u2019s\u2019 problem when several different population varlances are involved. Biometrika 34(1\u20132):28\u201335","journal-title":"Biometrika"},{"key":"10363_CR59","doi-asserted-by":"crossref","unstructured":"Yang T, Meng Z, Hao J, Zhang C, Zheng Y, Zheng Z (2018) Towards efficient detection and optimal response against sophisticated opponents. arXiv preprint arXiv:1809.04240","DOI":"10.24963\/ijcai.2019\/88"},{"key":"10363_CR60","doi-asserted-by":"crossref","unstructured":"Yang T, Hao J, Meng Z, Zheng Y, Zhang C, Zheng Z (2019) Bayes-tomop: A fast detection and best response algorithm towards sophisticated opponents. In: AAMAS, pp 2282\u20132284","DOI":"10.24963\/ijcai.2019\/88"},{"issue":"1","key":"10363_CR61","doi-asserted-by":"publisher","first-page":"957","DOI":"10.1007\/s11063-019-09999-3","volume":"50","author":"C Zhang","year":"2019","unstructured":"Zhang C, Zhang Y, Shi X, Almpanidis G, Fan G, Shen X (2019) On incremental learning for gradient boosting decision trees. Neural Process Lett 50(1):957\u2013987","journal-title":"Neural Process Lett"},{"key":"10363_CR62","doi-asserted-by":"crossref","unstructured":"Zheng Y, Xie X, Su T, Ma L, Hao J, Meng Z, Liu Y, Shen R, Chen Y, Fan C (2019) Wuji: Automatic online combat game testing using evolutionary deep reinforcement learning. In: 2019 34th IEEE\/ACM International Conference on Automated Software Engineering (ASE), IEEE, pp 772\u2013784","DOI":"10.1109\/ASE.2019.00077"},{"issue":"4","key":"10363_CR63","doi-asserted-by":"publisher","first-page":"366","DOI":"10.1145\/267580.267590","volume":"29","author":"H Zhu","year":"1997","unstructured":"Zhu H, Hall PA, May JH (1997) Software unit test coverage and adequacy. ACM Computing Surveys (CSUR) 29(4):366\u2013427","journal-title":"ACM Computing Surveys (CSUR)"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-023-10363-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-023-10363-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-023-10363-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,10,4]],"date-time":"2023-10-04T12:17:41Z","timestamp":1696421861000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-023-10363-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,8,24]]},"references-count":63,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2023,9]]}},"alternative-id":["10363"],"URL":"https:\/\/doi.org\/10.1007\/s10664-023-10363-2","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,8,24]]},"assertion":[{"value":"27 June 2023","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"24 August 2023","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declared that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"111"}}