{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,21]],"date-time":"2026-05-21T19:37:19Z","timestamp":1779392239702,"version":"3.53.1"},"reference-count":68,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2017,10,3]],"date-time":"2017-10-03T00:00:00Z","timestamp":1506988800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2018,6]]},"DOI":"10.1007\/s10664-017-9553-x","type":"journal-article","created":{"date-parts":[[2017,10,3]],"date-time":"2017-10-03T06:08:05Z","timestamp":1507010885000},"page":"1490-1518","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":15,"title":["Empirical study on the discrepancy between performance testing results from virtual and physical environments"],"prefix":"10.1007","volume":"23","author":[{"given":"Muhammad Moiz","family":"Arif","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Weiyi","family":"Shang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Emad","family":"Shihab","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2017,10,3]]},"reference":[{"key":"9553_CR1","doi-asserted-by":"crossref","unstructured":"Ahmed TM, Bezemer CP, Chen TH, Hassan AE, Shang W (2016) Studying the effectiveness of application performance management (apm) tools for detecting performance regressions for web applications: an experience report. In: MSR 2016: proceedings of the 13th working conference on mining software repositories","DOI":"10.1145\/2901739.2901774"},{"key":"9553_CR2","unstructured":"Andale (2012) Statistics how to - coefficient of determination (r squared). http:\/\/www.statisticshowto.com\/what-is-a-coefficient-of-determination\/ . Accessed: 2017-04-04"},{"key":"9553_CR3","unstructured":"Apache (2007) Tomcat. http:\/\/tomcat.apache.org\/ . Accessed: 2015-06-01"},{"key":"9553_CR4","unstructured":"Apache (2008) Jmeter. http:\/\/jmeter.apache.org\/ . Accessed: 2015-06-01"},{"key":"9553_CR5","unstructured":"Blackberry (2014) Blackberry enterprise server. https:\/\/ca.blackberry.com\/enterprise . Accessed: 2017-04-04"},{"key":"9553_CR6","unstructured":"Bod\u00edk P, Goldszmidt M, Fox A (2008) Hilighter: automatically building robust signatures of performance behavior for small- and large-scale systems. In: Proceedings of the third conference on tackling computer systems problems with machine learning techniques, SysML\u201908, pp 3\u20133"},{"key":"9553_CR7","doi-asserted-by":"crossref","unstructured":"Brosig F, Gorsler F, Huber N, Kounev S (2013) Evaluating approaches for performance prediction in virtualized environments. In: 2013 IEEE 21st international symposium on modelling, analysis and simulation of computer and telecommunication systems. IEEE, pp 404\u2013408","DOI":"10.1109\/MASCOTS.2013.61"},{"key":"9553_CR8","unstructured":"CA Technologies (2011) The avoidable cost of downtime. http:\/\/www3.ca.com\/~\/media\/files\/articles\/avoidable_cost_of_downtime_part_2_ita.aspx"},{"key":"9553_CR9","doi-asserted-by":"publisher","first-page":"317","DOI":"10.1007\/978-3-642-50096-1_48","volume-title":"Compstat","author":"J. Chambers","year":"1990","unstructured":"Chambers J, Hastie T, Pregibon D (1990) Statistical models in S. In: Compstat: proceedings in computational statistics, 9th symposium held at Dubrovnik, Yugoslavia, 1990. Physica-Verlag HD, Heidelberg, pp 317\u2013321"},{"key":"9553_CR10","unstructured":"Chen PM, Noble BD (2001) When virtual is better than real [operating system relocation to virtual machines]. In: Proceedings of the eighth workshop on hot topics in operating systems, 2001, pp 133\u2013138"},{"key":"9553_CR11","doi-asserted-by":"crossref","unstructured":"Cito J, Leitner P, Fritz T, Gall HC (2015) The making of cloud applications: an empirical study on software development for the cloud. In: Proceedings of the 2015 10th joint meeting on foundations of software engineering, ESEC\/FSE 2015, pp 393\u2013403","DOI":"10.1145\/2786805.2786826"},{"key":"9553_CR12","unstructured":"CloudScale-Project (2014) Cloudstore. https:\/\/github.com\/cloudscale-project\/cloudstore . Accessed: 2015-06-01"},{"key":"9553_CR13","unstructured":"Cohen I, Goldszmidt M, Kelly T, Symons J, Chase JS (2004) Correlating instrumentation data to system states: a building block for automated diagnosis and control. In: Proceedings of the 6th conference on symposium on operating systems design & implementation, OSDI\u201904, vol 6, pp 16\u201316"},{"key":"9553_CR14","doi-asserted-by":"crossref","unstructured":"Cohen I, Zhang S, Goldszmidt M, Symons J, Kelly T, Fox A (2005) Capturing, indexing, clustering, and retrieving system history. In: Proceedings of the twentieth ACM symposium on operating systems principles, SOSP \u201905, pp 105\u2013118","DOI":"10.1145\/1095810.1095821"},{"key":"9553_CR15","unstructured":"Costantini D (2015) How to configure a pass-through disk with hyper-v. http:\/\/thesolving.com\/virtualization\/how-to-configure-a-pass-through-disk-with-hyper-v\/ . Accessed: 2017-04-04"},{"key":"9553_CR16","doi-asserted-by":"publisher","first-page":"74","DOI":"10.1145\/2408776.2408794","volume":"56","author":"J Dean","year":"2013","unstructured":"Dean J, Barroso LA (2013) The tail at scale. Commun ACM 56:74\u201380","journal-title":"Commun ACM"},{"key":"9553_CR17","unstructured":"Dee (2014) Performance-testing systems on virtual machines that normally run on physical machines. http:\/\/sqa.stackexchange.com\/questions\/7709\/performance-testing-systems-on-virtual-machines-that-normally-run-on-physical-ma . Accessed: 2017-04-04"},{"key":"9553_CR18","unstructured":"Eeton K (2012) How one second could cost amazon $1.6 billion in sales. http:\/\/www.fastcompany.com\/1825005\/how-one-second-could-cost-amazon-16-billion-sales . Accessed: 2016-03-11"},{"key":"9553_CR19","doi-asserted-by":"crossref","unstructured":"Foo KC, Jiang ZM, Adams B, Hassan AE, Zou Y, Flora P (2010) Mining performance regression testing repositories for automated performance analysis. In: 10th international conference on quality software (QSIC), 2010, pp 32\u201341","DOI":"10.1109\/QSIC.2010.35"},{"key":"9553_CR20","doi-asserted-by":"crossref","unstructured":"Freedman D (2009) Statistical models: theory and practice. Cambridge University Press","DOI":"10.1017\/CBO9780511815867"},{"key":"9553_CR21","doi-asserted-by":"crossref","unstructured":"Harrell FE (2001) Regression modeling strategies: with applications to linear models, logistic regression, and survival analysis. Springer","DOI":"10.1007\/978-1-4757-3462-1"},{"key":"9553_CR22","doi-asserted-by":"crossref","unstructured":"Heger C, Happe J, Farahbod R (2013) Automated root cause isolation of performance regressions during software development. In: ICPE \u201913: proceedings of the 4th ACM\/SPEC international conference on performance engineering, pp 27\u201338","DOI":"10.1145\/2479871.2479879"},{"key":"9553_CR23","unstructured":"Huber N, von Quast M, Hauck M, Kounev S (2011) Evaluating and modeling virtualization performance overhead for cloud environments. In: Proceedings of the 1st international conference on cloud computing and services science, pp 563\u2013573"},{"key":"9553_CR24","unstructured":"Jaffe D, Muirhead T (2011) Dell dvd store. http:\/\/linux.dell.com\/dvdstore\/ . Accessed: 2015-06-01"},{"key":"9553_CR25","unstructured":"Jain R (1990) The art of computer systems performance analysis: techniques for experimental design, measurement, simulation, and modeling. Wiley"},{"key":"9553_CR26","doi-asserted-by":"crossref","unstructured":"Jiang M, Munawar M, Reidemeister T, Ward P (2009a) Automatic fault detection and diagnosis in complex software systems by information-theoretic monitoring. In: Proceedings of 2009 IEEE\/IFIP international conference on dependable systems networks, pp 285\u2013294","DOI":"10.1109\/DSN.2009.5270324"},{"key":"9553_CR27","doi-asserted-by":"crossref","unstructured":"Jiang M, Munawar MA, Reidemeister T, Ward PA (2009b) System monitoring with metric-correlation models: problems and solutions. In: Proceedings of the 6th international conference on autonomic computing, pp 13\u201322","DOI":"10.1145\/1555228.1555233"},{"key":"9553_CR28","unstructured":"Jiang ZM, Hassan AE, Hamann G, Flora P (2009) Automated performance analysis of load tests. In: IEEE International conference on software maintenance, 2009. ICSM 2009, pp 125\u2013134"},{"key":"9553_CR29","doi-asserted-by":"crossref","unstructured":"Jin G, Song L, Shi X, Scherpelz J, Lu S (2012) Understanding and detecting real-world performance bugs. In: Proceedings of the 33rd ACM SIGPLAN conference on programming language design and implementation, PLDI \u201912. ACM, pp 77\u201388","DOI":"10.1145\/2254064.2254075"},{"key":"9553_CR30","unstructured":"Kabacoff RI (2011) R in action. In: R in action. Manning Publications Co., Staten Island, NY , pp 207\u2013213"},{"key":"9553_CR31","unstructured":"Kearon S (2012) Can you use a virtual machine to performance test an application? http:\/\/stackoverflow.com\/questions\/8906954\/can-you-use-a-virtual-machine-to-performance-test-an-application . Accessed: 2017-04-04"},{"key":"9553_CR32","unstructured":"Kohavi R (1995) A study of cross-validation and bootstrap for accuracy estimation and model selection. In: Proceedings of the 14th international joint conference on artificial intelligence, IJCAI\u201995, vol 2, pp 1137\u20131143"},{"issue":"5","key":"9553_CR33","doi-asserted-by":"publisher","first-page":"295","DOI":"10.1145\/2020976.1958789","volume":"36","author":"S Kraft","year":"2011","unstructured":"Kraft S, Casale G, Krishnamurthy D, Greer D, Kilpatrick P (2011) Io performance prediction in consolidated virtualized environments. SIGSOFT Softw Eng Notes 36(5):295\u2013306","journal-title":"SIGSOFT Softw Eng Notes"},{"issue":"5","key":"9553_CR34","first-page":"1","volume":"28","author":"M Kuhn","year":"2008","unstructured":"Kuhn M (2008) Building predictive models in r using the caret package. J Stat Softw Articles 28(5):1\u201326","journal-title":"J Stat Softw Articles"},{"issue":"3","key":"9553_CR35","doi-asserted-by":"publisher","first-page":"15:1","DOI":"10.1145\/2885497","volume":"16","author":"P Leitner","year":"2016","unstructured":"Leitner P, Cito J (2016) Patterns in the chaos\u2014a study of performance variation and predictability in public iaas clouds. ACM Trans Internet Technol 16(3):15:1\u201315:23","journal-title":"ACM Trans Internet Technol"},{"key":"9553_CR36","doi-asserted-by":"crossref","unstructured":"Luo Q, Poshyvanyk D, Grechanik M (2016) Mining performance regression inducing code changes in evolving software. In: Proceedings of the 13th international conference on mining software repositories, MSR \u201916, pp 25\u201336","DOI":"10.1145\/2901739.2901765"},{"key":"9553_CR37","doi-asserted-by":"crossref","unstructured":"Malik H, Adams B, Hassan AE (2010a) Pinpointing the subsystems responsible for the performance deviations in a load test. In: 2010 IEEE 21st international symposium on software reliability engineering, pp 201\u2013210","DOI":"10.1109\/ISSRE.2010.43"},{"key":"9553_CR38","doi-asserted-by":"crossref","unstructured":"Malik H, Jiang ZM, Adams B, Hassan AE, Flora P, Hamann G (2010b) Automatic comparison of load tests to support the performance analysis of large enterprise systems. In: CSMR \u201910: proceedings of the 2010 14th European conference on software maintenance and reengineering, pp 222\u2013231","DOI":"10.1109\/CSMR.2010.39"},{"key":"9553_CR39","doi-asserted-by":"crossref","unstructured":"Malik H, Jiang ZM, Adams B, Hassan AE, Flora P, Hamann G (2010c) Automatic comparison of load tests to support the performance analysis of large enterprise systems. In: 2010 14th European conference on software maintenance and reengineering, pp 222\u2013231","DOI":"10.1109\/CSMR.2010.39"},{"key":"9553_CR40","doi-asserted-by":"crossref","unstructured":"Malik H, Hemmati H, Hassan AE (2013) Automatic detection of performance deviations in the load testing of large scale systems. In: 2013 35th international conference on software engineering (ICSE), pp 1012\u20131021","DOI":"10.1109\/ICSE.2013.6606651"},{"issue":"5","key":"9553_CR41","doi-asserted-by":"publisher","first-page":"2146","DOI":"10.1007\/s10664-015-9381-9","volume":"21","author":"S Mcintosh","year":"2016","unstructured":"Mcintosh S, Kamei Y, Adams B, Hassan AE (2016) An empirical study of the impact of modern code review practices on software quality. Empirical Softw Engg 21(5):2146\u20132189","journal-title":"Empirical Softw Engg"},{"key":"9553_CR42","doi-asserted-by":"crossref","unstructured":"Menon A, Santos JR, Turner Y, Janakiraman GJ, Zwaenepoel W (2005) Diagnosing performance overheads in the xen virtual machine environment. In: Proceedings of the 1st ACM\/USENIX international conference on virtual execution environments, pp 13\u201323","DOI":"10.1145\/1064979.1064984"},{"key":"9553_CR43","unstructured":"Merrill CL (2009) Load testing sugarcrm in a virtual machine. http:\/\/www.webperformance.com\/library\/reports\/virtualization2\/ . Accessed: 2017-04-04"},{"key":"9553_CR44","unstructured":"Microsoft Technet (2007) Windows performance counters. https:\/\/technet.microsoft.com\/en-us\/library\/cc780836(v=ws.10).aspx . Accessed: 2015-06-01"},{"key":"9553_CR45","doi-asserted-by":"crossref","unstructured":"Netto MA, Menon S, Vieira HV, Costa LT, De Oliveira FM, Saad R, Zorzo A (2011) Evaluating load generation in virtualized environments for software performance testing. In: IEEE International symposium on parallel and distributed processing workshops and phd forum (IPDPSW), 2011. IEEE, pp 993\u20131000","DOI":"10.1109\/IPDPS.2011.244"},{"key":"9553_CR46","doi-asserted-by":"crossref","unstructured":"Nguyen TH, Adams B, Jiang ZM, Hassan AE, Nasser M, Flora P (2012) Automated detection of performance regressions using statistical process control techniques. In: Proceedings of the 3rd ACM\/SPEC international conference on performance engineering, ICPE \u201912, pp 299\u2013310","DOI":"10.1145\/2188286.2188344"},{"key":"9553_CR47","doi-asserted-by":"crossref","unstructured":"Nistor A, Jiang T, Tan L (2013a) Discovering, reporting, and fixing performance bugs. In: 2013 10th working conference on mining software repositories (MSR), pp 237\u2013246","DOI":"10.1109\/MSR.2013.6624035"},{"key":"9553_CR48","doi-asserted-by":"crossref","unstructured":"Nistor A, Song L, Marinov D, Lu S (2013b) Toddler: detecting performance problems via similar memory-access patterns. In: Proceedings of the 2013 international conference on software engineering, ICSE \u201913. IEEE Press, Piscataway, NJ, USA, pp 562\u2013571","DOI":"10.1109\/ICSE.2013.6606602"},{"key":"9553_CR49","unstructured":"NIST\/SEMATECH (2003) E-handbook of statistical methods. http:\/\/www.itl.nist.gov\/div898\/handbook\/eda\/section3\/qqplot.htm . Accessed: 2015-06-01"},{"key":"9553_CR50","unstructured":"Oracle (1998) MYSQL Server 5.6. https:\/\/www.mysql.com\/ . Accessed: 2015-06-01"},{"key":"9553_CR51","doi-asserted-by":"crossref","unstructured":"Refaeilzadeh P, Tang L, Liu H (2009) Cross-validation. In: Encyclopedia of database systems. MA, Boston, pp 532\u2013538","DOI":"10.1007\/978-0-387-39940-9_565"},{"key":"9553_CR52","unstructured":"Rodola G (2009) Psutil. https:\/\/github.com\/giampaolo\/psutil . Accessed: 2015-06-01"},{"key":"9553_CR53","doi-asserted-by":"crossref","unstructured":"Shang W, Hassan AE, Nasser M, Flora P (2015) Automated detection of performance regressions using regression models on clustered performance counters. In: Proceedings of the 6th ACM\/SPEC international conference on performance engineering, ICPE \u201915, pp 15\u201326","DOI":"10.1145\/2668930.2688052"},{"key":"9553_CR54","unstructured":"Shewhart WA (1931) Economic control of quality of manufactured product, vol 509. ASQ Quality Press"},{"key":"9553_CR55","unstructured":"Srion E (2015) The time for hyper-v pass-through disks has passed. http:\/\/www.altaro.com\/hyper-v\/hyper-v-pass-through-disks\/ . Accessed: 2017-04-04"},{"key":"9553_CR56","doi-asserted-by":"crossref","unstructured":"Stapleton JH (2008) Models for probability and statistical inference: theory and applications. Wiley","DOI":"10.1002\/9780470183410"},{"key":"9553_CR57","unstructured":"SugarCRM (2017) Sugarcrm. https:\/\/www.sugarcrm.com\/ . Accessed: 2017-04-04"},{"key":"9553_CR58","doi-asserted-by":"crossref","unstructured":"Syer MD, Jiang ZM, Nagappan M, Hassan AE, Nasser M, Flora P (2013) Leveraging performance counters and execution logs to diagnose memory-related performance issues. In: 29th IEEE international conference on software maintenance (ICSM \u201913), pp 110\u2013119","DOI":"10.1109\/ICSM.2013.22"},{"issue":"1","key":"9553_CR59","doi-asserted-by":"publisher","first-page":"189","DOI":"10.1007\/s10515-016-0196-8","volume":"24","author":"MD Syer","year":"2017","unstructured":"Syer MD, Shang W, Jiang ZM, Hassan AE (2017) Continuous validation of performance test workloads. Autom Softw Eng 24(1):189\u2013231","journal-title":"Autom Softw Eng"},{"key":"9553_CR60","unstructured":"Tintin (2011) Performance test is not reliable on virtual machine? https:\/\/social.technet.microsoft.com\/forums\/windowsserver\/en-US\/06c0e09b-c5b4-4e2c-90e3-61b06483fe5b\/performance-test-is-not-reliable-on-virtual-machine?forum=winserverhyperv . Accessed: 2017-04-04"},{"key":"9553_CR61","unstructured":"TPC (2001) TPC-W. http:\/\/www.tpc.org\/tpcw . Accessed: 2015-06-01"},{"key":"9553_CR62","doi-asserted-by":"crossref","unstructured":"Tsakiltsidis S, Miranskyy A, Mazzawi E (2016) On automatic detection of performance bugs. In: 2016 IEEE international symposium on software reliability engineering workshops (ISSREW) , pp 132\u2013139","DOI":"10.1109\/ISSREW.2016.43"},{"key":"9553_CR63","unstructured":"Tyson J (2001) How network address translation works. http:\/\/computer.howstuffworks.com\/nat2.htm . Accessed: 2017-04-04"},{"key":"9553_CR64","unstructured":"VMWare (2016) Accelerate software development and testing with the vmware virtualization platform. http:\/\/www.vmware.com\/pdf\/development_testing.pdf . Accessed: 2016-03-16"},{"key":"9553_CR65","doi-asserted-by":"crossref","unstructured":"Walker HM (1929) Studies in the history of statistical method: with special reference to certain educational problems. Williams & Wilkins Co","DOI":"10.1037\/13379-000"},{"key":"9553_CR66","doi-asserted-by":"crossref","unstructured":"Woodside M, Franks G, Petriu DC (2007) The future of software performance engineering. In: Future of software engineering, 2007, pp 171\u2013187","DOI":"10.1109\/FOSE.2007.32"},{"key":"9553_CR67","doi-asserted-by":"crossref","unstructured":"Xiong P, Pu C, Zhu X, Griffith R (2013) Vperfguard: an automated model-driven framework for application performance diagnosis in consolidated cloud environments. In: Proceedings of the 4th ACM\/SPEC international conference on performance engineering, ICPE \u201913, pp 271\u2013282","DOI":"10.1145\/2479871.2479909"},{"key":"9553_CR68","doi-asserted-by":"crossref","unstructured":"Zaman S, Adams B, Hassan AE (2012) A qualitative study on performance bugs. In: 2012 9th IEEE working conference on mining software repositories (MSR), pp 199\u2013208","DOI":"10.1109\/MSR.2012.6224281"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10664-017-9553-x\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-017-9553-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-017-9553-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,10,18]],"date-time":"2020-10-18T16:27:48Z","timestamp":1603038468000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10664-017-9553-x"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,10,3]]},"references-count":68,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2018,6]]}},"alternative-id":["9553"],"URL":"https:\/\/doi.org\/10.1007\/s10664-017-9553-x","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2017,10,3]]},"assertion":[{"value":"3 October 2017","order":1,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}