{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T12:22:54Z","timestamp":1740140574995,"version":"3.37.3"},"reference-count":36,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2019,6,26]],"date-time":"2019-06-26T00:00:00Z","timestamp":1561507200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2019,6,26]],"date-time":"2019-06-26T00:00:00Z","timestamp":1561507200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"DOI":"10.13039\/100010663","name":"H2020 European Research Council","doi-asserted-by":"publisher","award":["H2020-ICT-2016-2 761493"],"award-info":[{"award-number":["H2020-ICT-2016-2 761493"]}],"id":[{"id":"10.13039\/100010663","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001659","name":"German Research Foundation","doi-asserted-by":"crossref","award":["SFB901"],"award-info":[{"award-number":["SFB901"]}],"id":[{"id":"10.13039\/501100001659","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J Wireless Com Network"],"published-print":{"date-parts":[[2019,12]]},"DOI":"10.1186\/s13638-019-1493-2","type":"journal-article","created":{"date-parts":[[2019,6,26]],"date-time":"2019-06-26T13:03:41Z","timestamp":1561554221000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["Automated testing of NFV orchestrators against carrier-grade multi-PoP scenarios using emulation-based smoke testing"],"prefix":"10.1186","volume":"2019","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3164-7635","authenticated-orcid":false,"given":"Manuel","family":"Peuster","sequence":"first","affiliation":[]},{"given":"Michael","family":"Marchetti","sequence":"additional","affiliation":[]},{"given":"Gerardo","family":"Garc\u00eda de Blas","sequence":"additional","affiliation":[]},{"given":"Holger","family":"Karl","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,6,26]]},"reference":[{"issue":"9","key":"1493_CR1","doi-asserted-by":"publisher","first-page":"1206","DOI":"10.1002\/ett.3084","volume":"27","author":"H. Karl","year":"2016","unstructured":"H. Karl, S. Dr\u00e4xler, M. Peuster, A. Galis, M. Bredel, A. Ramos, J. Martrat, M. S. Siddiqui, S. van Rossem, W. Tavernier, et al, DevOps for network function virtualisation: an architectural approach. Trans. Emerg. Telecommun. Technol.27(9), 1206\u20131215 (2016).","journal-title":"Trans. Emerg. Telecommun. Technol."},{"key":"1493_CR2","unstructured":"OpenStack Project, DevStack. Online at: https:\/\/docs.openstack.org\/devstack\/latest\/ . Accessed Aug 2018."},{"key":"1493_CR3","doi-asserted-by":"publisher","first-page":"973","DOI":"10.1109\/ICCW.2017.7962785","volume-title":"2017 IEEE International Conference on Communications Workshops (ICC Workshops)","author":"S. Dr\u00e4xler","year":"2017","unstructured":"S. Dr\u00e4xler, H. Karl, M. Peuster, H. R. Kouchaksaraei, M. Bredel, J. Lessmann, T. Soenen, W. Tavernier, S. Mendel-Brin, G. Xilouris, in 2017 IEEE International Conference on Communications Workshops (ICC Workshops). SONATA: Service programming and orchestration for virtualized software networks (IEEEParis, 2017), pp. 973\u2013978. https:\/\/doi.org\/10.1109\/ICCW.2017.7962785 ."},{"key":"1493_CR4","unstructured":"Linux Foundation, ONAP: Open Network Automation Platform. Online at: https:\/\/www.onap.org . Accessed Mar 2018."},{"key":"1493_CR5","unstructured":"ETSI OSM, Open Sorce MANO. Online at: https:\/\/osm.etsi.org . Accessed Feb 2018."},{"key":"1493_CR6","doi-asserted-by":"publisher","unstructured":"M. Peuster, M. Marchetti, G. G. de Blas, H. Karl, in 2018 European Conference on Networks and Communications (EuCNC). Emulation-based Smoke Testing of NFV Orchestrators in Large Multi-PoP Environments (Ljubljana, 2018), pp. 1\u20139. https:\/\/doi.org\/10.1109\/EuCNC.2018.8442701 .","DOI":"10.1109\/EuCNC.2018.8442701"},{"key":"1493_CR7","volume-title":"Automated software testing: introduction, management, and performance","author":"E. Dustin","year":"1999","unstructured":"E. Dustin, J. Rashka, J. Paul, Automated software testing: introduction, management, and performance (Addison-Wesley Longman Publishing Co., Inc., Boston, 1999)."},{"key":"1493_CR8","doi-asserted-by":"publisher","first-page":"148","DOI":"10.1109\/NFV-SDN.2016.7919490","volume-title":"2016 IEEE Conference on Network Function Virtualization and Software Defined Networks (NFV-SDN)","author":"M. Peuster","year":"2016","unstructured":"M. Peuster, H. Karl, S. van Rossem, in 2016 IEEE Conference on Network Function Virtualization and Software Defined Networks (NFV-SDN). MeDICINE: Rapid prototyping of production-ready network services in multi-PoP environments (IEEEPalo Alto, 2016), pp. 148\u2013153. https:\/\/doi.org\/10.1109\/NFV-SDN.2016.7919490 ."},{"key":"1493_CR9","unstructured":"ETSI, Network Functions Virtualisation (NFV) Release 2; Protocols and Data Models; RESTful protocols specification for the Os-Ma-nfvo Reference Point (2018). Website. Online at https:\/\/www.etsi.org\/deliver\/etsi_{g}s\/NFV-SOL\/001_099\/005\/02.04.01_60\/gs_NFV-SOL005v020401p.pdf . Accessed 18 June 2019."},{"key":"1493_CR10","unstructured":"M. Peuster, ETSI NFV SOL005 Test Suite. Online at: https:\/\/github.com\/mpeuster\/etsi-nfv-sol005-test-suite . Accessed Aug 2018."},{"key":"1493_CR11","doi-asserted-by":"publisher","unstructured":"C. Parada, J. Bonnet, E. Fotopoulou, A. Zafeiropoulos, E. Kapassa, M. Touloupou, D. Kyriazis, R. Vilalta, R. Mu\u00f1oz, R. Casellas, R. Mart\u00ednez, G. Xilouris, in 2018 European Conference on Networks and Communications (EuCNC). 5GTAGNO: A Beyond-Mano Service Platform, (2018), pp. 26\u201330. https:\/\/doi.org\/10.1109\/EuCNC.2018.8443232 .","DOI":"10.1109\/EuCNC.2018.8443232"},{"key":"1493_CR12","doi-asserted-by":"publisher","unstructured":"H. R. Kouchaksaraei, S. Dr\u00e4xler, M. Peuster, H. Karl, in 2018 European Conference on Networks and Communications (EuCNC). Programmable and flexible management and orchestration of virtualized network functions (Ljubljana, 2018), pp. 1\u20139. https:\/\/doi.org\/10.1109\/EuCNC.2018.8442528 .","DOI":"10.1109\/EuCNC.2018.8442528"},{"key":"1493_CR13","unstructured":"ETSI, GS NFV 002: Network Functions Virtualisation (NFV): Architectural Framework (2014). Online at https:\/\/www.etsi.org\/deliver\/etsi_gs\/nfv\/001_099\/002\/01.02.01_60\/gs_nfv002v010201p.pdf . Accessed 18 June 2019."},{"key":"1493_CR14","first-page":"144","volume":"4","author":"S. McConnell","year":"1996","unstructured":"S. McConnell, Daily build and smoke test. IEEE Softw.4:, 144\u2013143 (1996).","journal-title":"IEEE Softw."},{"key":"1493_CR15","unstructured":"Linux Foundation, OPNFV Project (Website. Online at https:\/\/www.opnfv.org . Accessed 18 June 2019."},{"key":"1493_CR16","doi-asserted-by":"publisher","first-page":"307","DOI":"10.1109\/EuCNC.2016.7561053","volume-title":"2016 European Conference on Networks and Communications (EuCNC)","author":"A. F. Cattoni","year":"2016","unstructured":"A. F. Cattoni, G. C. Madue\u00f1o, M. Dieudonne, P. Merino, A. D. Zayas, A. Salmeron, F. Carlier, B. Saint Germain, D. Morris, R. Figueiredo, et al, in 2016 European Conference on Networks and Communications (EuCNC). An end-to-end testing ecosystem for 5GIEEEAthens, 2016), pp. 307\u2013312. https:\/\/doi.org\/10.1109\/EuCNC.2016.7561053 ."},{"key":"1493_CR17","doi-asserted-by":"publisher","unstructured":"M. Zhao, F. L. Gall, P. Cousin, R. Vilalta, R. Mu\u00f1oz, S. Castro, M. Peuster, S. Schneider, M. Siapera, E. Kapassa, D. Kyriazis, P. Hasselmeyer, G. Xilouris, C. Tranoris, S. Denazis, J. Martrat, in 2017 IEEE Conference on Network Function Virtualization and Software Defined Networks (NFV-SDN). Verification and validation framework for 5g network services and apps, (2017), pp. 321\u2013326. https:\/\/doi.org\/10.1109\/NFV-SDN.2017.8169878 .","DOI":"10.1109\/NFV-SDN.2017.8169878"},{"key":"1493_CR18","doi-asserted-by":"crossref","unstructured":"L. Cao, P. Sharma, S. Fahmy, V. Saxena, in Network Function Virtualization and Software Defined Network (NFV-SDN), 2015 IEEE Conference On. NFV-VITAL: a framework for characterizing the performance of virtual Network Functions (IEEE, 2015), pp. 93\u201399.","DOI":"10.1109\/NFV-SDN.2015.7387412"},{"issue":"9","key":"1493_CR19","doi-asserted-by":"publisher","first-page":"110","DOI":"10.1109\/MCOM.2017.1700127","volume":"55","author":"R. V. Rosa","year":"2017","unstructured":"R. V. Rosa, C. Bertoldo, C. E. Rothenberg, Take your vnf to the gym: a testing framework for automated nfv performance benchmarking. IEEE Commun. Mag.55(9), 110\u2013117 (2017). https:\/\/doi.org\/10.1109\/MCOM.2017.1700127 .","journal-title":"IEEE Commun. Mag."},{"key":"1493_CR20","doi-asserted-by":"publisher","unstructured":"M. Peuster, H. Karl, in 2017 IEEE Conference on Network Function Virtualization and Software Defined Networks (NFVSDN). Profile your chains, not functions: Automated network service profiling in DevOps environments (Berlin, 2017), pp. 1\u20136. https:\/\/doi.org\/10.1109\/NFV-SDN.2017.8169826 .","DOI":"10.1109\/NFV-SDN.2017.8169826"},{"key":"1493_CR21","doi-asserted-by":"publisher","first-page":"469","DOI":"10.1145\/2486001.2491689","volume-title":"Proceedings of the ACM SIGCOMM 2013 conference on SIGCOMM(SIGCOMM \u201913)","author":"M. Keller","year":"2013","unstructured":"M. Keller, C. Robbert, M. Peuster, in Proceedings of the ACM SIGCOMM 2013 conference on SIGCOMM(SIGCOMM \u201913). An evaluation testbed for adaptive, topology-aware deployment of elastic applications (ACMNew York, 2013), pp. 469\u2013470. https:\/\/doi.org\/10.1145\/2486001.2491689 ."},{"issue":"3","key":"1493_CR22","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1145\/956993.956995","volume":"33","author":"B. Chun","year":"2003","unstructured":"B. Chun, D. Culler, T. Roscoe, A. Bavier, L. Peterson, M. Wawrzoniak, M. Bowman, PlanetLab: an overlay testbed for broad-coverage services. SIGCOMM Comput. Commun. Rev.33(3), 3\u201312 (2003). https:\/\/doi.org\/10.1145\/956993.956995 .","journal-title":"SIGCOMM Comput. Commun. Rev."},{"key":"1493_CR23","unstructured":"H, 2020 SoftFIRE consortium, SoftFIRE approach to experiment management: why and how. Online at https:\/\/goo.gl\/LxLfLz . Accessed Aug 2018."},{"key":"1493_CR24","unstructured":"H, 2020 Fed4Fire+ consortium, Fed4Fire: The largest federation of testbeds in Europe. Online at: https:\/\/www.fed4fire.eu . Accessed Aug 2018."},{"key":"1493_CR25","doi-asserted-by":"publisher","unstructured":"B. Lantz, B. Heller, N. McKeown, in Proceedings of the 9th ACM SIGCOMM Workshop on Hot Topics in Networks(Hotnets-IX). A network in a laptop: rapid prototyping for softwaredefined networks (ACMNew York, 2010), p. 6. https:\/\/doi.org\/10.1145\/1868447.1868466 .","DOI":"10.1145\/1868447.1868466"},{"key":"1493_CR26","doi-asserted-by":"publisher","unstructured":"J. Ahrenholz, C. Danilov, T. R. Henderson, J. H. Kim, in MILCOM 2008 - 2008 IEEE Military Communications Conference. CORE: A real-time network emulator (San Diego, 2008), pp. 1\u20137. https:\/\/doi.org\/10.1109\/MILCOM.2008.4753614 .","DOI":"10.1109\/MILCOM.2008.4753614"},{"issue":"4","key":"1493_CR27","doi-asserted-by":"publisher","first-page":"166","DOI":"10.1109\/MCOM.2015.7081091","volume":"53","author":"L. Mamatas","year":"2015","unstructured":"L. Mamatas, S. Clayman, A. Galis, A service-aware virtualized software-defined infrastructure. Commun. Mag. IEEE. 53(4), 166\u2013174 (2015).","journal-title":"Commun. Mag. IEEE"},{"key":"1493_CR28","unstructured":"Jenkins Project, Jenkins CI webpage. Online at: https:\/\/jenkins.io . Accessed Apr 2018."},{"key":"1493_CR29","unstructured":"M. Peuster, Containernet. Online at: https:\/\/containernet.github.io . Accessed Aug 2018."},{"issue":"9","key":"1493_CR30","doi-asserted-by":"publisher","first-page":"1765","DOI":"10.1109\/JSAC.2011.111002","volume":"29","author":"Simon Knight","year":"2011","unstructured":"S. Knight, H. X. Nguyen, N. Falkner, R. Bowden, M. Roughan, The internet topology zoo. Sel. Areas Commun. IEEE J.29(9), 1765\u20131775 (2011). https:\/\/doi.org\/10.1109\/JSAC.2011.111002 .","journal-title":"IEEE Journal on Selected Areas in Communications"},{"key":"1493_CR31","unstructured":"G. Bernini, M. Zhao, E. Kraja, Specialist Task Force 557: NFV API Conformance test specification. Online at: https:\/\/portal.etsi.org\/STF\/STFs\/STFHomePages\/STF557 . Accessed Apr 2019."},{"key":"1493_CR32","unstructured":"Canonical Ltd, Juju Charms. Online at: https:\/\/www.jujucharms.com . Accessed Aug 2018."},{"key":"1493_CR33","unstructured":"SONATA Consortium, SONATA Project. Online at: http:\/\/sonata-nfv.eu . Accessed Jan 2018."},{"key":"1493_CR34","unstructured":"5GTANGO Consortium, 5GTANGO Project. Online at: http:\/\/5gtango.eu . Accessed Aug 2018."},{"key":"1493_CR35","unstructured":"ETSI OSM, OSM vim-emu code repository. Online at: https:\/\/osm.etsi.org\/gitweb\/?p=osm\/vim-emu.git; . Accessed Aug 2018."},{"key":"1493_CR36","unstructured":"ETSI OSM, OSM vim-emu documentation. Online at: https:\/\/goo.gl\/XZNLVw . Accessed Aug 2018."}],"container-title":["EURASIP Journal on Wireless Communications and Networking"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1186\/s13638-019-1493-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1186\/s13638-019-1493-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1186\/s13638-019-1493-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,9,22]],"date-time":"2022-09-22T00:12:22Z","timestamp":1663805542000},"score":1,"resource":{"primary":{"URL":"https:\/\/jwcn-eurasipjournals.springeropen.com\/articles\/10.1186\/s13638-019-1493-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,6,26]]},"references-count":36,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2019,12]]}},"alternative-id":["1493"],"URL":"https:\/\/doi.org\/10.1186\/s13638-019-1493-2","relation":{},"ISSN":["1687-1499"],"issn-type":[{"type":"electronic","value":"1687-1499"}],"subject":[],"published":{"date-parts":[[2019,6,26]]},"assertion":[{"value":"19 September 2018","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"10 June 2019","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"26 June 2019","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare that they have no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"172"}}