{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,21]],"date-time":"2026-04-21T14:50:57Z","timestamp":1776783057038,"version":"3.51.2"},"publisher-location":"New York, NY, USA","reference-count":52,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,11,7]],"date-time":"2020-11-07T00:00:00Z","timestamp":1604707200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"European Research Council - Advanced Grant (ERC-AdG)","award":["787703"],"award-info":[{"award-number":["787703"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020,11,8]]},"DOI":"10.1145\/3368089.3409730","type":"proceedings-article","created":{"date-parts":[[2020,11,8]],"date-time":"2020-11-08T06:03:52Z","timestamp":1604815432000},"page":"876-888","source":"Crossref","is-referenced-by-count":114,"title":["Model-based exploration of the frontier of behaviours for deep learning system testing"],"prefix":"10.1145","author":[{"given":"Vincenzo","family":"Riccio","sequence":"first","affiliation":[{"name":"USI Lugano, Switzerland"}]},{"given":"Paolo","family":"Tonella","sequence":"additional","affiliation":[{"name":"USI Lugano, Switzerland"}]}],"member":"320","published-online":{"date-parts":[[2020,11,7]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/2970276.2970311"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180160"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238192"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/378456.378511"},{"key":"e_1_3_2_2_5_1","unstructured":"BeamNG GmbH. [n.d.]. BeamNG.research. https:\/\/www.beamng.gmbh\/research  BeamNG GmbH. [n.d.]. BeamNG.research. https:\/\/www.beamng.gmbh\/research"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.3758\/s13428-011-0081-0"},{"key":"e_1_3_2_2_7_1","volume-title":"Davide Del Testa","author":"Bojarski Mariusz","year":"2016"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1016\/B978-0-12-079050-0.50020-5"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.312"},{"key":"e_1_3_2_2_10_1","unstructured":"C. J. Clopper and E. S. Pearson. 1934. The Use of Confidence or Fiducial Limits Illustrated in the Case of the Binomial. Biometrika 26 4 ( 12 1934 ) 404-413. https:\/\/doi.org\/10.1093\/biomet\/26.4. 404 10.1093\/biomet\/26.4.404  C. J. Clopper and E. S. Pearson. 1934. The Use of Confidence or Fiducial Limits Illustrated in the Case of the Binomial. Biometrika 26 4 ( 12 1934 ) 404-413. https:\/\/doi.org\/10.1093\/biomet\/26.4. 404 10.1093\/biomet\/26.4.404"},{"key":"e_1_3_2_2_11_1","volume-title":"Genetic and Evolutionary Computation-GECCO","author":"de Jong Edwin D.","year":"2004"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/4235.996017"},{"key":"e_1_3_2_2_13_1","volume-title":"Breakthroughs in statistics","author":"Fisher Ronald Aylmer"},{"key":"e_1_3_2_2_14_1","unstructured":"International Organization for Standardization (ISO ). 2019. ISO\/PAS 21448: Road vehicles-Safety of the intended functionality.  International Organization for Standardization (ISO ). 2019. ISO\/PAS 21448: Road vehicles-Safety of the intended functionality."},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.5555\/2503308.2503311"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330566"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.5555\/3086952"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3264835"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2379776.2379787"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-94-015-1288-6"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753357"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380395"},{"key":"e_1_3_2_2_23_1","unstructured":"ISO. 2011. Road vehicles-Functional safety.  ISO. 2011. Road vehicles-Functional safety."},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357127"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/1276958.1277175"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.5555\/265006"},{"key":"e_1_3_2_2_28_1","first-page":"2278","volume-title":"Proc. IEEE 86","author":"LeCun Yann","year":"1998"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1162\/EVCO_a_00025"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2001576.2001606"},{"key":"e_1_3_2_2_31_1","volume-title":"Soviet physics doklady","author":"Levenshtein Vladimir I"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2016.09.015"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2931037.2931054"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2016.26"},{"key":"e_1_3_2_2_36_1","volume-title":"New Horizons in Evolutionary Robotics, St\u00e9phane Doncieux, Nicolas Bred\u00e8che, and Jean-Baptiste Mouret (Eds.)","author":"Mouret Jean-Baptiste"},{"key":"e_1_3_2_2_37_1","unstructured":"Jean-Baptiste Mouret and Jef Clune. 2015. Illuminating search spaces by mapping elites. arXiv:1504.04909 [cs.AI]  Jean-Baptiste Mouret and Jef Clune. 2015. Illuminating search spaces by mapping elites. arXiv:1504.04909 [cs.AI]"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.2307\/2344614"},{"key":"e_1_3_2_2_39_1","volume-title":"AASHTO Green Book (GDHS-7)-A Policy on Geometric Design of Highways and Streets","author":"American Association of State Highway and Transportation Oficials. 2018."},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2017.2663435"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2013.13"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.3758\/s13428-013-0434-y"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_2_44_1","unstructured":"Vincenzo Riccio Gunel Jahangirova Andrea Stocco Nargiz Humbatova Michael Weiss and Paolo Tonella. 2020. Testing Machine Learning based Systems: A Systematic Mapping. Empirical Software Engineering ( 2020 ). https:\/\/doi.org\/10. 1007\/s10664-020-09881-0 10.1007\/s10664-020-09881-0  Vincenzo Riccio Gunel Jahangirova Andrea Stocco Nargiz Humbatova Michael Weiss and Paolo Tonella. 2020. Testing Machine Learning based Systems: A Systematic Mapping. Empirical Software Engineering ( 2020 ). https:\/\/doi.org\/10. 1007\/s10664-020-09881-0 10.1007\/s10664-020-09881-0"},{"key":"e_1_3_2_2_45_1","unstructured":"P. Selinger. 2003. Potrace: a polygon-based tracing algorithm. ( 2003 ). http:\/\/potrace.sourceforge.net\/potrace.pdf  P. Selinger. 2003. Potrace: a polygon-based tracing algorithm. ( 2003 ). http:\/\/potrace.sourceforge.net\/potrace.pdf"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_2_47_1","unstructured":"DeepJanus 2019. DeepJanus: A Tool for Model-based Exploration of the Frontier of Behaviours for Deep Learning Systems Testing. https:\/\/github.com\/ testingautomated-usi\/DeepJanus.  DeepJanus 2019. DeepJanus: A Tool for Model-based Exploration of the Frontier of Behaviours for Deep Learning Systems Testing. https:\/\/github.com\/ testingautomated-usi\/DeepJanus."},{"key":"e_1_3_2_2_48_1","unstructured":"Unity Technologies. 2019. Unity. https:\/\/unity.com  Unity Technologies. 2019. Unity. https:\/\/unity.com"},{"key":"e_1_3_2_2_49_1","volume-title":"Feature-Guided Black-Box Safety Testing of Deep Neural Networks. In Tools and Algorithms for the Construction and Analysis of Systems-24th International Conference, TACAS. 408-426","author":"Wicker Matthew","year":"2018"},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/1273463.1273483"},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2009.11.706"},{"key":"e_1_3_2_2_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"}],"event":{"name":"ESEC\/FSE '20: 28th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","location":"Virtual Event USA","acronym":"ESEC\/FSE '20","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3368089.3409730","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3368089.3409730","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:01:57Z","timestamp":1750197717000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3368089.3409730"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,11,7]]},"references-count":52,"alternative-id":["10.1145\/3368089.3409730","10.1145\/3368089"],"URL":"https:\/\/doi.org\/10.1145\/3368089.3409730","relation":{},"subject":[],"published":{"date-parts":[[2020,11,7]]}}}