{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,26]],"date-time":"2025-09-26T04:50:30Z","timestamp":1758862230066,"version":"3.40.4"},"publisher-location":"Berlin, Heidelberg","reference-count":39,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"type":"print","value":"9783642367267"},{"type":"electronic","value":"9783642367274"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2013]]},"DOI":"10.1007\/978-3-642-36727-4_12","type":"book-chapter","created":{"date-parts":[[2013,2,4]],"date-time":"2013-02-04T13:36:02Z","timestamp":1359984962000},"page":"173-188","source":"Crossref","is-referenced-by-count":47,"title":["Benchmarking in the Cloud: What It Should, Can, and Cannot Be"],"prefix":"10.1007","author":[{"given":"Enno","family":"Folkerts","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Alexander","family":"Alexandrov","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kai","family":"Sachs","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Alexandru","family":"Iosup","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Volker","family":"Markl","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Cafer","family":"Tosun","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","reference":[{"key":"12_CR1","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"18","DOI":"10.1007\/978-3-642-10424-4_3","volume-title":"Performance Evaluation and Benchmarking","author":"K. Huppler","year":"2009","unstructured":"Huppler, K.: The Art of Building a Good Benchmark. In: Nambiar, R., Poess, M. (eds.) TPCTC 2009. LNCS, vol.\u00a05895, pp. 18\u201330. Springer, Heidelberg (2009)"},{"key":"12_CR2","doi-asserted-by":"crossref","unstructured":"Binnig, C., Kossmann, D., Kraska, T., Loesing, S.: How is the weather tomorrow?: towards a benchmark for the cloud. In: DB Test. ACM (2009)","DOI":"10.1145\/1594156.1594168"},{"key":"12_CR3","unstructured":"SPEC: The SPEC CPU2006 Benchmark, http:\/\/www.spec.org\/cpu2006\/"},{"key":"12_CR4","unstructured":"TPC: The TPC-C Benchmark, http:\/\/www.tpc.org\/tpcc\/"},{"issue":"1","key":"12_CR5","doi-asserted-by":"publisher","first-page":"43","DOI":"10.1145\/1558334.1558339","volume":"38","author":"D. Florescu","year":"2009","unstructured":"Florescu, D., Kossmann, D.: Rethinking cost and performance of database systems. SIGMOD Record\u00a038(1), 43\u201348 (2009)","journal-title":"SIGMOD Record"},{"key":"12_CR6","unstructured":"Gray, J. (ed.): The Benchmark Handbook for Database and Transaction Systems, 2nd edn. Morgan Kaufmann (1993)"},{"key":"12_CR7","unstructured":"Kounev, S.: Performance Engineering of Distributed Component-Based Systems - Benchmarking, Modeling and Performance Prediction. PhD thesis, Technische Universit\u00e4t Darmstadt (2005)"},{"issue":"8","key":"12_CR8","doi-asserted-by":"publisher","first-page":"410","DOI":"10.1016\/j.peva.2009.01.003","volume":"66","author":"K. Sachs","year":"2009","unstructured":"Sachs, K., Kounev, S., Bacon, J., Buchmann, A.: Performance evaluation of message-oriented middleware using the SPECjms 2007 benchmark. Performance Evaluation\u00a066(8), 410\u2013434 (2009)","journal-title":"Performance Evaluation"},{"key":"12_CR9","doi-asserted-by":"crossref","unstructured":"Sachs, K.: Performance Modeling and Benchmarking of Event-Based Systems. PhD thesis, TU Darmstadt (2011)","DOI":"10.1145\/2188286.2188314"},{"key":"12_CR10","doi-asserted-by":"crossref","unstructured":"Madeira, H., Vieira, M., Sachs, K., Kounev, S.: Dagstuhl Seminar 10292. In: Resilience Benchmarking, Springer (2011)","DOI":"10.1007\/978-3-642-29032-9_14"},{"key":"12_CR11","unstructured":"NIST: The NIST Definition of Cloud Computing (2011), http:\/\/csrc.nist.gov\/publications\/nistpubs\/800-145\/SP800-145.pdf"},{"key":"12_CR12","doi-asserted-by":"crossref","unstructured":"Youseff, L., Butrico, M., Silva, D.D.: Towards a unified ontology of cloud computing. In: Proc. of the Grid Computing Environments Workshop (GCE 2008) (2008)","DOI":"10.1109\/GCE.2008.4738443"},{"key":"12_CR13","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"97","DOI":"10.1007\/978-3-642-32627-1_7","volume-title":"Topics in Performance Evaluation, Measurement and Characterization","author":"K. Huppler","year":"2012","unstructured":"Huppler, K.: Benchmarking with Your Head in the Cloud. In: Nambiar, R., Poess, M. (eds.) TPCTC 2011. LNCS, vol.\u00a07144, pp. 97\u2013110. Springer, Heidelberg (2012)"},{"key":"12_CR14","unstructured":"Leimeister, S., B\u00f6hm, M., Riedl, C., Krcmar, H.: The business perspective of cloud computing: Actors, roles and value networks. In: Alexander, P.M., Turpin, M., van Deventer, J.P. (eds.) ECIS (2010)"},{"key":"12_CR15","unstructured":"SPEC Open Systems Group: Report on cloud computing to the OSG Steering Committee. Technical Report OSG-wg-final-20120214 (February 2012)"},{"key":"12_CR16","doi-asserted-by":"crossref","unstructured":"Shen, S., Visser, O., Iosup, A.: Rtsenv: An experimental environment for real-time strategy games. In: Shirmohammadi, S., Griwodz, C. (eds.) NETGAMES, pp. 1\u20136. IEEE (2011)","DOI":"10.1109\/NetGames.2011.6080981"},{"issue":"3","key":"12_CR17","doi-asserted-by":"publisher","first-page":"380","DOI":"10.1109\/TPDS.2010.82","volume":"22","author":"V. Nae","year":"2011","unstructured":"Nae, V., Iosup, A., Prodan, R.: Dynamic resource provisioning in massively multiplayer online games. IEEE Trans. Parallel Distrib. Syst.\u00a022(3), 380\u2013395 (2011)","journal-title":"IEEE Trans. Parallel Distrib. Syst."},{"issue":"5","key":"12_CR18","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1109\/MIC.2010.57","volume":"14","author":"S. Ratti","year":"2010","unstructured":"Ratti, S., Hariri, B., Shirmohammadi, S.: A survey of first-person shooter gaming traffic on the internet. IEEE Internet Computing\u00a014(5), 60\u201369 (2010)","journal-title":"IEEE Internet Computing"},{"key":"12_CR19","unstructured":"Li, M., Sasanka, R., Adve, S., Chen, Y., Debes, E.: The ALPBench benchmark suite for complex multimedia applications. In: Proceedings of the IEEE International Workload Characterization Symposium, pp. 34\u201345 (2005)"},{"key":"12_CR20","unstructured":"Lee, C., Potkonjak, M., Mangione-Smith, W.H.: Mediabench: A tool for evaluating and synthesizing multimedia and communicatons systems. In: MICRO, pp. 330\u2013335 (1997)"},{"key":"12_CR21","doi-asserted-by":"crossref","unstructured":"Guthaus, M.R., Ringenberg, J.S., Ernst, D., Austin, T.M., Mudge, T., Brown, R.B.: MiBench: A free, commercially representative embedded benchmark suite. In: Proceedings of the Fourth Annual IEEE International Workshop on Workload Characterization, WWC-4 (Cat. No. 01EX538), pp. 3\u201314. IEEE (2001)","DOI":"10.1109\/WWC.2001.990739"},{"key":"12_CR22","doi-asserted-by":"crossref","unstructured":"Fritsch, T., Ritter, H., Schiller, J.H.: The effect of latency and network limitations on mmorpgs: a field study of everquest2. In: NETGAMES, pp. 1\u20139. ACM (2005)","DOI":"10.1145\/1103599.1103623"},{"issue":"11","key":"12_CR23","doi-asserted-by":"publisher","first-page":"34","DOI":"10.1145\/1167838.1167859","volume":"49","author":"K.T. Chen","year":"2006","unstructured":"Chen, K.T., Huang, P., Lei, C.L.: How sensitive are online gamers to network quality? Commun. ACM\u00a049(11), 34\u201338 (2006)","journal-title":"Commun. ACM"},{"issue":"1","key":"12_CR24","doi-asserted-by":"publisher","first-page":"52","DOI":"10.1016\/j.comnet.2005.04.008","volume":"49","author":"M. Claypool","year":"2005","unstructured":"Claypool, M.: The effect of latency on user performance in real-time strategy games. Computer Networks\u00a049(1), 52\u201370 (2005)","journal-title":"Computer Networks"},{"key":"12_CR25","doi-asserted-by":"crossref","unstructured":"Beigbeder, T., Coughlan, R., Lusher, C., Plunkett, J., Agu, E., Claypool, M.: The effects of loss and latency on user performance in unreal tournament 2003. In: Chang Feng, W. (ed.) NETGAMES, pp. 144\u2013151. ACM (2004)","DOI":"10.1145\/1016540.1016556"},{"key":"12_CR26","doi-asserted-by":"crossref","unstructured":"Balint, M., Posea, V., Dimitriu, A., Iosup, A.: User behavior, social networking, and playing style in online and face to face bridge communities. In: NETGAMES, pp. 1\u20132. IEEE (2010)","DOI":"10.1109\/NETGAMES.2010.5679543"},{"key":"12_CR27","series-title":"SCI","doi-asserted-by":"publisher","first-page":"303","DOI":"10.1007\/978-3-642-20344-2_12","volume-title":"Next Generation Data Technologies for Collective Computational Intelligence","author":"A. Iosup","year":"2011","unstructured":"Iosup, A., L\u0103sc\u0103teu, A.: Clouds and Continuous Analytics Enabling Social Networks for Massively Multiplayer Online Games. In: Bessis, N., Xhafa, F. (eds.) Next Generation Data Technologies for Collective Computational Intelligence. SCI, vol.\u00a0352, pp. 303\u2013328. Springer, Heidelberg (2011)"},{"key":"12_CR28","doi-asserted-by":"publisher","first-page":"11","DOI":"10.1109\/ICPADS.2008.70","volume-title":"Proceedings of the 2008 14th IEEE International Conference on Parallel and Distributed Systems, ICPADS 2008","author":"K. Kim","year":"2008","unstructured":"Kim, K., Jeon, K., Han, H., Kim, S.G., Jung, H., Yeom, H.Y.: Mrbench: A benchmark for mapreduce framework. In: Proceedings of the 2008 14th IEEE International Conference on Parallel and Distributed Systems, ICPADS 2008, pp. 11\u201318. IEEE Computer Society, Washington, DC (2008)"},{"key":"12_CR29","doi-asserted-by":"crossref","unstructured":"Chen, Y., Ganapathi, A., Griffith, R., Katz, R.H.: The case for evaluating mapreduce performance using workload suites. In: MASCOTS, pp. 390\u2013399. IEEE (2011)","DOI":"10.1109\/MASCOTS.2011.12"},{"key":"12_CR30","doi-asserted-by":"crossref","unstructured":"Pavlo, A., Paulson, E., Rasin, A., Abadi, D.J., DeWitt, D.J., Madden, S., Stonebraker, M.: A comparison of approaches to large-scale data analysis. In: \u00c7etintemel, U., Zdonik, S.B., Kossmann, D., Tatbul, N. (eds.) SIGMOD Conference, pp. 165\u2013178. ACM (2009)","DOI":"10.1145\/1559845.1559865"},{"key":"12_CR31","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"73","DOI":"10.1007\/978-3-642-18206-8_6","volume-title":"Performance Evaluation, Measurement and Characterization of Complex Systems","author":"K. Huppler","year":"2011","unstructured":"Huppler, K.: Price and the TPC. In: Nambiar, R., Poess, M. (eds.) TPCTC 2010. LNCS, vol.\u00a06417, pp. 73\u201384. Springer, Heidelberg (2011)"},{"key":"12_CR32","doi-asserted-by":"publisher","first-page":"579","DOI":"10.1145\/1807167.1807231","volume-title":"Proceedings of the 2010 ACM SIGMOD International Conference on Management of Data, SIGMOD 2010","author":"D. Kossmann","year":"2010","unstructured":"Kossmann, D., Kraska, T., Loesing, S.: An evaluation of alternative architectures for transaction processing in the cloud. In: Proceedings of the 2010 ACM SIGMOD International Conference on Management of Data, SIGMOD 2010, pp. 579\u2013590. ACM, New York (2010)"},{"key":"12_CR33","doi-asserted-by":"crossref","unstructured":"Islam, S., Lee, K., Fekete, A., Liu, A.: How a consumer can measure elasticity for cloud platforms. In: [38], pp. 85-96","DOI":"10.1145\/2188286.2188301"},{"key":"12_CR34","doi-asserted-by":"crossref","unstructured":"Rabl, T., Poess, M.: Parallel data generation for performance analysis of large, complex rdbms. In: Graefe, G., Salem, K. (eds.) DBTest, p. 5. ACM (2011)","DOI":"10.1145\/1988842.1988847"},{"key":"12_CR35","doi-asserted-by":"crossref","unstructured":"Frank, M., Poess, M., Rabl, T.: Efficient update data generation for dbms benchmarks. In: [38], pp. 169\u2013180","DOI":"10.1145\/2188286.2188315"},{"issue":"4","key":"12_CR36","doi-asserted-by":"publisher","first-page":"50","DOI":"10.1145\/1721654.1721672","volume":"53","author":"M. Armbrust","year":"2010","unstructured":"Armbrust, M., Fox, A., Griffith, R., Joseph, A.D., Katz, R.H., Konwinski, A., Lee, G., Patterson, D.A., Rabkin, A., Stoica, I., Zaharia, M.: A view of cloud computing. Commun. ACM\u00a053(4), 50\u201358 (2010)","journal-title":"Commun. ACM"},{"key":"12_CR37","doi-asserted-by":"crossref","unstructured":"Villegas, D., Antoniou, A., Sadjadi, S.M., Iosup, A.: An analysis of provisioning and allocation policies for infrastructure-as-a-service clouds. In: CCGRID (2012)","DOI":"10.1109\/CCGrid.2012.46"},{"key":"12_CR38","doi-asserted-by":"crossref","unstructured":"Iosup, A., Yigitbasi, N., Epema, D.H.J.: On the performance variability of production cloud services. In: CCGRID, pp. 104\u2013113. IEEE (2011)","DOI":"10.1109\/CCGrid.2011.22"},{"key":"12_CR39","unstructured":"Kaeli, D.R., Rolia, J., John, L.K., Krishnamurthy, D. (eds.): Third Joint WOSP\/SIPEW International Conference on Performance Engineering, ICPE 2012, Boston, MA, USA, April 22-25. ACM (2012)"}],"container-title":["Lecture Notes in Computer Science","Selected Topics in Performance Evaluation and Benchmarking"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-642-36727-4_12","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,29]],"date-time":"2025-04-29T19:24:48Z","timestamp":1745954688000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-642-36727-4_12"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013]]},"ISBN":["9783642367267","9783642367274"],"references-count":39,"URL":"https:\/\/doi.org\/10.1007\/978-3-642-36727-4_12","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2013]]}}}