{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,15]],"date-time":"2026-01-15T18:41:12Z","timestamp":1768502472717,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":36,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T00:00:00Z","timestamp":1726012800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Norges Forskningsr\u00e5d","award":["324674"],"award-info":[{"award-number":["324674"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,9,11]]},"DOI":"10.1145\/3650212.3680382","type":"proceedings-article","created":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T11:44:25Z","timestamp":1726055065000},"page":"1567-1578","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Policy Testing with MDPFuzz (Replicability Study)"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-3519-5514","authenticated-orcid":false,"given":"Quentin","family":"Mazouni","sequence":"first","affiliation":[{"name":"Simula Research Laboratory, Oslo, Norway"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2494-4279","authenticated-orcid":false,"given":"Helge","family":"Spieker","sequence":"additional","affiliation":[{"name":"Simula Research Laboratory, Oslo, Norway"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8980-7585","authenticated-orcid":false,"given":"Arnaud","family":"Gotlieb","sequence":"additional","affiliation":[{"name":"Simula Research Laboratory, Oslo, Norway"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1483-3858","authenticated-orcid":false,"given":"Mathieu","family":"Acher","sequence":"additional","affiliation":[{"name":"Univ Rennes, CNRS, Inria, IRISA, Institut Universitaire de France (IUF), Rennes, France"}]}],"member":"320","published-online":{"date-parts":[[2024,9,11]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Greg Brockman Vicki Cheung Ludwig Pettersson Jonas Schneider John Schulman Jie Tang and Wojciech Zaremba. 2016. OpenAI Gym. arxiv:arXiv:1606.01540."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1613\/jair.1.12716"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.2517-6161.1977.tb01600.x"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1613\/jair.639"},{"key":"e_1_3_2_1_5_1","volume-title":"Automatic Metamorphic Test Oracles for Action-Policy Testing. In International Conference on Automated Planning and Scheduling. https:\/\/api.semanticscholar.org\/CorpusID:259638200","author":"Eisenhut Jan","year":"2023","unstructured":"Jan Eisenhut, \u00c1lvaro Torralba, Maria Christakis, and J\u00f6rg Hoffmann. 2023. Automatic Metamorphic Test Oracles for Action-Policy Testing. In International Conference on Automated Planning and Scheduling. https:\/\/api.semanticscholar.org\/CorpusID:259638200"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3533767.3534392"},{"key":"e_1_3_2_1_7_1","volume-title":"The Cambridge Handbook of Artificial Intelligence, Keith Frankish and William M","unstructured":"2014. The Cambridge Handbook of Artificial Intelligence, Keith Frankish and William M. Ramsey (Eds.). Cambridge University Press, Cambridge, UK. isbn:978-0-521-87142-6"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.2307\/2982840"},{"key":"e_1_3_2_1_9_1","volume-title":"Gimitest: A framework for evaluating reinforcement learning agents. https:\/\/github.com\/DennisGross\/gimitest GitHub repository","author":"Gross Dennis","year":"2024","unstructured":"Dennis Gross, Quentin Mazouni, and Helge Spieker. 2024. Gimitest: A framework for evaluating reinforcement learning agents. https:\/\/github.com\/DennisGross\/gimitest GitHub repository"},{"key":"e_1_3_2_1_10_1","volume-title":"Curiosity-Driven Testing for Sequential Decision-Making Process. In 2024 IEEE\/ACM 46th International Conference on Software Engineering (ICSE). IEEE Computer Society","author":"He J.","unstructured":"J. He, Z. Yang, J. Shi, C. Yang, K. Kim, B. Xu, X. Zhou, and D. Lo. 2024. Curiosity-Driven Testing for Sequential Decision-Making Process. In 2024 IEEE\/ACM 46th International Conference on Software Engineering (ICSE). IEEE Computer Society, Los Alamitos, CA, USA. 949\u2013949. issn:1558-1225 https:\/\/doi.ieeecomputersociety.org\/"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11694"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"crossref","unstructured":"Rushang Karia and Siddharth Srivastava. 2020. Learning Generalized Relational Heuristic Networks for Model-Agnostic Planning. CoRR.","DOI":"10.1609\/aaai.v35i9.16983"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE56229.2023.00153"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2022.3150788"},{"key":"e_1_3_2_1_15_1","unstructured":"Quentin Mazouni Helge Spieker Arnaud Gotlieb and Mathieu Acher. 2023. A Review of Validation and Verification of Neural Network-based Policies for Sequential Decision Making. In Rencontres des Jeunes Chercheurs en Intelligence Artificielle (RJCIA). https:\/\/pfia23.icube.unistra.fr\/conferences\/rjcia\/Actes\/RJCIA2023_paper_5.pdf"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","unstructured":"Quentin Mazouni Helge Spieker Arnaud Gotlieb and Mathieu Acher. 2024. Policy Testing with MDPFuzz (Replicability Study). https:\/\/doi.org\/10.5281\/zenodo.12668777 10.5281\/zenodo.12668777","DOI":"10.5281\/zenodo.12668777"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3644032.3644458"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/96267.96279"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/IRC.2019.00120"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3533767.3534388"},{"key":"e_1_3_2_1_21_1","unstructured":"Joelle Pineau. ICSE\u201919 Keynote. Building Reproducible Reusable and Robust Machine Learning Software. https:\/\/2019.icse-conferences.org\/details\/icse-2019-Plenary-Sessions\/20\/Building-Reproducible-Reusable-and-Robust-Machine-Learning-Software"},{"key":"e_1_3_2_1_22_1","unstructured":"Qi Pang. 2023 2024. Inquiry about MDPFuzz. Personal communication. Email correspondence"},{"key":"e_1_3_2_1_23_1","unstructured":"Antonin Raffin. 2020. RL Baselines3 Zoo. https:\/\/github.com\/DLR-RM\/rl-baselines3-zoo"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","unstructured":"David Silver Thomas Hubert Julian Schrittwieser Ioannis Antonoglou Matthew Lai Arthur Guez Marc Lanctot Laurent Sifre Dharshan Kumaran Thore Graepel Timothy Lillicrap Karen Simonyan and Demis Hassabis. 2018. A general reinforcement learning algorithm that masters chess shogi and Go through self-play. Science https:\/\/doi.org\/10.1126\/science.aar6404 10.1126\/science.aar6404","DOI":"10.1126\/science.aar6404"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1609\/icaps.v32i1.19820"},{"key":"e_1_3_2_1_26_1","volume-title":"Reinforcement learning: An introduction","author":"Sutton Richard S","unstructured":"Richard S Sutton and Andrew G Barto. 2018. Reinforcement learning: An introduction. MIT press."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10515-022-00337-x"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2022\/72"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.8127026"},{"key":"e_1_3_2_1_30_1","volume-title":"Asnets: Deep learning for generalised planning. Journal of Artificial Intelligence Research, 68","author":"Toyer Sam","year":"2020","unstructured":"Sam Toyer, Sylvie Thi\u00e9baux, Felipe Trevizan, and Lexing Xie. 2020. Asnets: Deep learning for generalised planning. Journal of Artificial Intelligence Research, 68 (2020)."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00155"},{"key":"e_1_3_2_1_32_1","first-page":"2579","article-title":"Visualizing Data using t-SNE","volume":"9","author":"van der Maaten Laurens","year":"2008","unstructured":"Laurens van der Maaten and Geoffrey E. Hinton. 2008. Visualizing Data using t-SNE. Journal of Machine Learning Research, 9 (2008), 2579\u20132605. https:\/\/api.semanticscholar.org\/CorpusID:5855042","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE59848.2023.00041"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF00992698"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","unstructured":"Jin Zhang and Jingyue Li. 2020. Testing and verification of neural-network-based safety-critical control software: A systematic literature review. Information and Software Technology https:\/\/doi.org\/10.1016\/j.infsof.2020.106296 10.1016\/j.infsof.2020.106296","DOI":"10.1016\/j.infsof.2020.106296"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2023.3269804"}],"event":{"name":"ISSTA '24: 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis","location":"Vienna Austria","acronym":"ISSTA '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","AITO"]},"container-title":["Proceedings of the 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650212.3680382","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3650212.3680382","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:58:09Z","timestamp":1750294689000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650212.3680382"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,11]]},"references-count":36,"alternative-id":["10.1145\/3650212.3680382","10.1145\/3650212"],"URL":"https:\/\/doi.org\/10.1145\/3650212.3680382","relation":{},"subject":[],"published":{"date-parts":[[2024,9,11]]},"assertion":[{"value":"2024-09-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}