{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,19]],"date-time":"2025-11-19T14:42:46Z","timestamp":1763563366889},"reference-count":33,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2020,2,25]],"date-time":"2020-02-25T00:00:00Z","timestamp":1582588800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,2,25]],"date-time":"2020-02-25T00:00:00Z","timestamp":1582588800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"name":"European Union\u2019s H2020 research and innovation Programme","award":["644400"],"award-info":[{"award-number":["644400"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2020,5]]},"DOI":"10.1007\/s10664-020-09800-3","type":"journal-article","created":{"date-parts":[[2020,3,6]],"date-time":"2020-03-06T13:19:32Z","timestamp":1583500772000},"page":"2025-2054","update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":18,"title":["The virtual lands of Oz: testing an agribot in simulation"],"prefix":"10.1007","volume":"25","author":[{"given":"Cl\u00e9ment","family":"Robert","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Thierry","family":"Sotiropoulos","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"H\u00e9l\u00e8ne","family":"Waeselynck","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"J\u00e9r\u00e9mie","family":"Guiochet","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Simon","family":"Vernhes","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2020,2,25]]},"reference":[{"key":"9800_CR1","doi-asserted-by":"crossref","unstructured":"Andrews AA, Abdelgawad M, Gario A (2016) World model for testing urban search and rescue (USAR) robots using petri nets. In: Proceedings of the 4rd International Conference on Model-Driven Engineering and Software Development (MODELSWARD 2016), Rome, Italy, pp 663\u2013670","DOI":"10.5220\/0005782106630670"},{"key":"9800_CR2","doi-asserted-by":"crossref","unstructured":"Arnold J, Alexander R (2013) Testing autonomous robot control software using procedural content generation. In: Computer Safety, Reliability, and Security (SAFECOMP 2013), Toulouse, France, pp 33\u201344","DOI":"10.1007\/978-3-642-40793-2_4"},{"key":"9800_CR3","doi-asserted-by":"crossref","unstructured":"Bach J, Langner J, Otten S, Sax E, Holz\u00e4pfel M (2017) Test scenario selection for system-level verification and validation of geolocation-dependent automotive control systems. In: 2017 International conference on engineering, technology and innovation (ICE\/ITMC 2017), Madeira Island, Portugal, pp 203\u2013210","DOI":"10.1109\/ICE.2017.8279890"},{"key":"9800_CR4","doi-asserted-by":"crossref","unstructured":"Ben Abdessalem R, Nejati S, Briand LC, Stifter T (2016) Testing advanced driver assistance systems using multi-objective search and neural networks. In: Proceedings of the 31st IEEE\/ACM International Conference on Automated Software Engineering, Singapore, Singapore, pp 63\u201374","DOI":"10.1145\/2970276.2970311"},{"key":"9800_CR5","doi-asserted-by":"crossref","unstructured":"Ben Abdessalem R, Nejati S, Briand LC, Stifter T (2018) Testing vision-based control systems using learnable evolutionary algorithms. In: 2018 IEEE\/ACM 40th International Conference on Software Engineering (ICSE 2018), Gothenburg, Sweden, pp 1016\u20131026","DOI":"10.1145\/3180155.3180160"},{"key":"9800_CR6","doi-asserted-by":"crossref","unstructured":"Chen TY (2015) Metamorphic testing: a simple method for alleviating the test oracle problem. In: Proceedings of the 10th International Workshop on Automation of Software Test, Florence, Italy, pp 53\u201354","DOI":"10.1109\/AST.2015.18"},{"key":"9800_CR7","doi-asserted-by":"crossref","unstructured":"Echeverria G, Lassabe N, Degroote A, Lemaignan S (2011) Modular open robots simulation engine: Morse. In: IEEE international conference on robotics and automation (ICRA 2011), Shanghai, China","DOI":"10.1109\/ICRA.2011.5980252"},{"issue":"3","key":"9800_CR8","doi-asserted-by":"publisher","first-page":"183","DOI":"10.1049\/iet-its.2012.0188","volume":"8","author":"S Geyer","year":"2014","unstructured":"Geyer S, Kienle M, Franz B, Winner H, Bengler K, Baltzer M, Flemisch F, Kauer M, Wei\u00dfgerber T, Bruder R, Hakuli S, Meier S (2014) Concept and development of a unified ontology for generating test and use-case catalogues for assisted and automated vehicle guidance. IET Intelligent Transport Systems 8(3):183\u2013189","journal-title":"IET Intelligent Transport Systems"},{"key":"9800_CR9","doi-asserted-by":"publisher","unstructured":"Hallerbach S, Xia Y, Eberle U, Koester F (2018) Simulation-based identification of critical scenarios for cooperative and automated vehicles. Tech. rep., SAE International in United States. https:\/\/doi.org\/10.4271\/2018-01-1066","DOI":"10.4271\/2018-01-1066"},{"key":"9800_CR10","doi-asserted-by":"crossref","unstructured":"Klueck F, Li Y, Nica M, Tao J, Wotawa F (2018) Using ontologies for test suites generation for automated and autonomous driving functions. In: 2018 IEEE international symposium on software reliability engineering workshops (ISSREW 2018), Memphis, TN, USA, vol 00, pp 118\u2013123","DOI":"10.1109\/ISSREW.2018.00-20"},{"key":"9800_CR11","doi-asserted-by":"crossref","unstructured":"Koenig N, Howard A (2004) Design and use paradigms for gazebo, an open-source multi-robot simulator. In: 2004 IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS 2004), Sendai, Japan, vol 3, pp 2149\u20132154","DOI":"10.1109\/IROS.2004.1389727"},{"key":"9800_CR12","doi-asserted-by":"crossref","unstructured":"Lamprecht A, Ganslmeier T (2010) Simulation process for vehicle applications depending on alternative driving routes between real-world locations. Advanced Microsystems for Automotive Applications 2010, pp 377\u2013386","DOI":"10.1007\/978-3-642-16362-3_36"},{"key":"9800_CR13","doi-asserted-by":"crossref","unstructured":"Lindvall M, Porter A, Magnusson G, Schulze C (2017) Metamorphic model-based testing of autonomous systems. In: 2nd IEEE\/ACM International Workshop on Metamorphic Testing (ICSE 2017), Buenos Aires, Argentina, pp 35\u201341","DOI":"10.1109\/MET.2017.6"},{"key":"9800_CR14","doi-asserted-by":"crossref","unstructured":"Menghi C, Nejati S, Gaaloul K, Briand LC (2019) Generating automated and online test oracles for simulink models with continuous and uncertain behaviors. In: 27th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering, Tallinn, Estonia, pp 27\u201338","DOI":"10.1145\/3338906.3338920"},{"key":"9800_CR15","doi-asserted-by":"crossref","unstructured":"Micskei Z, Szatm\u00e1ri Z, Ol\u00e1h J, Majzik I (2012) A concept for testing robustness and safety of the context-aware behaviour of autonomous systems. In: Agent and Multi-Agent Systems. Technologies and Applications, Dubrovnik, Croatia, pp 504\u2013513","DOI":"10.1007\/978-3-642-30947-2_55"},{"key":"9800_CR16","doi-asserted-by":"crossref","unstructured":"Nentwig M, Stamminger M (2010) A method for the reproduction of vehicle test drives for the simulation based evaluation of image processing algorithms. In: 13th International IEEE Conference on Intelligent Transportation Systems, Madeira Island, Portugal, pp 1307\u20131312","DOI":"10.1109\/ITSC.2010.5625005"},{"key":"9800_CR17","unstructured":"Nguyen CD, Perini A, Tonella P, Miles S, Harman M, Luck M (2009) Evolutionary testing of autonomous software agents. In: Proceedings of The 8th international conference on autonomous agents and multiagent systems (AAMAS 2009), Budapest, Hungary, vol 1, pp 521\u2013528"},{"key":"9800_CR18","unstructured":"Okdal Sydac (2018) https:\/\/www.oktalsydac.com\/, Accessed 2019-09-19"},{"key":"9800_CR19","unstructured":"PreScan Simulation platform for ADAS (2018) https:\/\/tass.plm.automation.siemens.com\/prescan, Accessed 2019-09-19"},{"key":"9800_CR20","doi-asserted-by":"crossref","unstructured":"Sotiropoulos T, Guiochet J, Ingrand F, Weaselynck H (2016) Virtual worlds for testing robot navigation: a study on the difficulty level. In: IEEE 12th European on Dependable Computing Conference (EDCC 2016), Iasi, Romania, pp 153\u2013160","DOI":"10.1109\/EDCC.2016.14"},{"key":"9800_CR21","doi-asserted-by":"crossref","unstructured":"Sotiropoulos T, Waeselynck H, Guiochet J, Ingrand F (2017) Can robot navigation bugs be found in simulation? an exploratory study. In: 2017 IEEE International Conference on Software Quality, Reliability and Security (QRS 2017), Prague, Czech Republic, pp 150\u2013159","DOI":"10.1109\/QRS.2017.25"},{"key":"9800_CR22","doi-asserted-by":"crossref","unstructured":"Tian Y, Pei K, Jana S, Ray B (2018) Deeptest: Automated testing of deep-neural-network-driven autonomous cars. In: Proceedings of the 40th International Conference on Software Engineering (ICSE 2018), Gothenburg, Sweden, pp 303\u2013314","DOI":"10.1145\/3180155.3180220"},{"key":"9800_CR23","doi-asserted-by":"crossref","unstructured":"Timperley CS, Afzal A, Katz DS, Hernandez JM, Goues CL (2018) Crashing simulated planes is cheap: Can simulation detect robotics bugs early? In: 2018 IEEE 11th International Conference on Software Testing, Verification and Validation (ICST 2018), V\u00e4steras, Sweden, pp 331\u2013342","DOI":"10.1109\/ICST.2018.00040"},{"issue":"3","key":"9800_CR24","doi-asserted-by":"publisher","first-page":"172","DOI":"10.1109\/TCIAIG.2011.2148116","volume":"3","author":"J Togelius","year":"2011","unstructured":"Togelius J, Yannakakis GN, Stanley KO, Browne C (2011) Search-based procedural content generation: a taxonomy and survey. IEEE Transactions on Computational Intelligence and AI in Games 3(3):172\u2013186","journal-title":"IEEE Transactions on Computational Intelligence and AI in Games"},{"key":"9800_CR25","unstructured":"Tractica (2016) Agricultural robots \u2013 executive summary. Research Report. https:\/\/www.tractica.com\/research\/agricultural-robots\/, Accessed 2019-09-19"},{"key":"9800_CR26","doi-asserted-by":"crossref","unstructured":"Ulbrich S, Menzel T, Reschka A, Schuldt F, Maurer M (2015) Defining and substantiating the terms scene, situation, and scenario for automated driving. In: 2015 IEEE 18th International Conference on Intelligent Transportation Systems, Gran Canaria, Spain, pp 982\u2013988","DOI":"10.1109\/ITSC.2015.164"},{"key":"9800_CR27","unstructured":"Unreal Game Engine (2018) https:\/\/www.unrealengine.com\/, Accessed 2019-09-19"},{"key":"9800_CR28","unstructured":"Virtual Test Drive (2018) http:\/\/www.mscsoftware.com\/product\/virtual-test-drive, Accessed 2019-09-19"},{"key":"9800_CR29","doi-asserted-by":"crossref","unstructured":"Zendel O, Herzner W, Murschitz M (2013) Vitro - model based vision testing for robustness. In: IEEE International Symposium on Robotics (ISR 2013), Seoul, Korea, pp 1\u20136","DOI":"10.1109\/ISR.2013.6695667"},{"issue":"1","key":"9800_CR30","doi-asserted-by":"publisher","first-page":"95","DOI":"10.1007\/s11263-017-1020-z","volume":"125","author":"O Zendel","year":"2017","unstructured":"Zendel O, Murschitz M, Humenberger M, Herzner W (2017) How good is my test data? introducing safety analysis for computer vision. Int J Comput Vis 125(1):95\u2013109","journal-title":"Int J Comput Vis"},{"key":"9800_CR31","doi-asserted-by":"crossref","unstructured":"Zhang M, Zhang Y, Zhang L, Liu C, Khurshid S (2018) Deeproad: Gan-based metamorphic testing and input validation framework for autonomous driving systems. In: Proceedings of the 33rd ACM\/IEEE International Conference on Automated Software Engineering, Montpellier, France, pp 132\u2013142","DOI":"10.1145\/3238147.3238187"},{"key":"9800_CR32","unstructured":"Zofka MR, Kuhnt F, Kohlhaas R, Rist C, Schamm T, Z\u00f6llner JM (2015) Data-driven simulation and parametrization of traffic scenarios for the development of advanced driver assistance systems. In: 2015 18th International Conference on Information Fusion (Fusion 2015), Washington, DC, USA, pp 1422\u20131428"},{"key":"9800_CR33","doi-asserted-by":"crossref","unstructured":"Zou X, Alexander R, McDermid J (2014) Safety validation of sense and avoid algorithms using simulation and evolutionary search. In: Computer Safety, Reliability, and Security, (SAFECOMP 2014), Florence, Italy, pp 33\u201348","DOI":"10.1007\/978-3-319-10506-2_3"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-020-09800-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10664-020-09800-3\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-020-09800-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,2,24]],"date-time":"2021-02-24T00:46:02Z","timestamp":1614127562000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10664-020-09800-3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,2,25]]},"references-count":33,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2020,5]]}},"alternative-id":["9800"],"URL":"https:\/\/doi.org\/10.1007\/s10664-020-09800-3","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,2,25]]},"assertion":[{"value":"25 February 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}