{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,21]],"date-time":"2026-05-21T02:51:16Z","timestamp":1779331876906,"version":"3.51.4"},"reference-count":83,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"8","license":[{"start":{"date-parts":[[2021,8,1]],"date-time":"2021-08-01T00:00:00Z","timestamp":1627776000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2021,8,1]],"date-time":"2021-08-01T00:00:00Z","timestamp":1627776000000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2021,8,1]],"date-time":"2021-08-01T00:00:00Z","timestamp":1627776000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2021,8,1]],"date-time":"2021-08-01T00:00:00Z","timestamp":1627776000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001729","name":"Stiftelsen f\u00f6r\u00a0Strategisk Forskning","doi-asserted-by":"publisher","award":["GMT14-0032"],"award-info":[{"award-number":["GMT14-0032"]}],"id":[{"id":"10.13039\/501100001729","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100003077","name":"Knowledge Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100003077","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001659","name":"Deutsche Forschungsgemeinschaft","doi-asserted-by":"publisher","award":["KO 3445\/11-1"],"award-info":[{"award-number":["KO 3445\/11-1"]}],"id":[{"id":"10.13039\/501100001659","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["1836752"],"award-info":[{"award-number":["1836752"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Wallenberg Foundation"},{"name":"Swedish strategic research programme eSSENCE"},{"name":"Google Faculty Research Award"},{"name":"National Science and Engineering Research Council of Canada"},{"DOI":"10.13039\/501100011688","name":"Electronic Components and Systems for European Leadership","doi-asserted-by":"publisher","award":["783162"],"award-info":[{"award-number":["783162"]}],"id":[{"id":"10.13039\/501100011688","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Dutch NWO Vidi"},{"DOI":"10.13039\/100004682","name":"Oracle","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100004682","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Intel Labs"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IIEEE Trans. Software Eng."],"published-print":{"date-parts":[[2021,8,1]]},"DOI":"10.1109\/tse.2019.2927908","type":"journal-article","created":{"date-parts":[[2019,7,10]],"date-time":"2019-07-10T20:18:02Z","timestamp":1562789882000},"page":"1528-1543","source":"Crossref","is-referenced-by-count":67,"title":["Methodological Principles for Reproducible Performance Evaluation in Cloud Computing"],"prefix":"10.1109","volume":"47","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1364-8127","authenticated-orcid":false,"given":"Alessandro Vittorio","family":"Papadopoulos","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6999-7297","authenticated-orcid":false,"given":"Laurens","family":"Versluis","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5582-8812","authenticated-orcid":false,"given":"Andre","family":"Bauer","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3462-6426","authenticated-orcid":false,"given":"Nikolas","family":"Herbst","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Joakim von","family":"Kistowski","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2774-9284","authenticated-orcid":false,"given":"Ahmed","family":"Ali-Eldin","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9263-673X","authenticated-orcid":false,"given":"Cristina L.","family":"Abad","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9943-1809","authenticated-orcid":false,"given":"Jose Nelson","family":"Amaral","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7035-2322","authenticated-orcid":false,"given":"Petr","family":"Tuma","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8030-9398","authenticated-orcid":false,"given":"Alexandru","family":"Iosup","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref73","first-page":"1","article-title":"The seven deadly sins of cloud computing research","author":"schwarzkopf","year":"2012","journal-title":"Proc 4th USENIX Conf Hot Topics Cloud Comput"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-36727-4_12"},{"key":"ref71","article-title":"Evaluation of measurement data–guide to the expression of uncertainty in measurement","year":"2008"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.6028\/NIST.TN.1297"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1007\/11605300_13"},{"key":"ref77","first-page":"232","article-title":"On grid performance evaluation using synthetic workloads","author":"iosup","year":"2006","journal-title":"Proc Workshop Job Scheduling Strategies Parallel Process"},{"key":"ref74","first-page":"864","article-title":"Collective Knowledge: Towards R&D Sustainability","author":"grigori fursin","year":"2016","journal-title":"Design Automation Test in Europe Conference Exhibition (DATE)"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-43659-3_1"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1007\/11777830_5"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/TC.2006.85"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-15277-1_46"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1016\/j.comcom.2010.09.014"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.14778\/1920841.1920902"},{"key":"ref32","volume":"1","author":"easton","year":"1997","journal-title":"Statistics Glossary"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1145\/2994031"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/CGO.2007.32"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/MM.2007.56"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1145\/3030207.3030229"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1145\/2885497"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/CCGrid.2011.22"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2016.127"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1145\/3164537"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/SEAMS.2015.23"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2018.2870389"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1002\/spe.2382"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.14236\/ewic\/EASE2008.8"},{"key":"ref27","first-page":"1","article-title":"The Flexlab approach to realistic evaluation of networked systems","author":"ricci","year":"2007","journal-title":"Proc 4th USENIX Conf Netw Syst Des Implementation"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-45735-6_1"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1177\/001316446002000104"},{"key":"ref29","first-page":"1","article-title":"Queues don’t matter when you can JUMP them!","author":"grosvenor","year":"2015","journal-title":"Proc 12th USENIX Conf Netw Syst Des Implementation"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1037\/h0031619"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1037\/h0026256"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.2307\/2529310"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1145\/2807591.2807644"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pcbi.1003285"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/3236332"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1145\/2812803"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1038\/533452a"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/2413176.2413206"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1145\/1113361.1113368"},{"key":"ref26","first-page":"215","article-title":"An experimentation workbench for replayable networking research","author":"eide","year":"2007","journal-title":"Proc 4th USENIX Conf Netw Syst Des Implementation"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1145\/2723872.2723885"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1145\/2930659"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1145\/2983574"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2009.84"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1109\/ISPASS.2018.00029"},{"key":"ref57","first-page":"41","article-title":"SPEC CPU2017–Next-generation compute benchmark","author":"bucek","year":"2018","journal-title":"Proc ACM\/SPEC Int Conf Perform Eng"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1145\/2668930.2688819"},{"key":"ref55","article-title":"SPEC cloud IaaS 2018 benchmark design overview","year":"2018","journal-title":"Tech Rep"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/71.149962"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1145\/127695.122760"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1145\/2414740.2414747"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2015.03.007"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2008.98"},{"key":"ref40","first-page":"1","article-title":"Rampant software errors may undermine scientific results","volume":"3","author":"soergela","year":"2015","journal-title":"F1000Res"},{"key":"ref12","article-title":"High performance computing evaluation A methodology based on scientific application requirements","author":"ferro","year":"2014","journal-title":"CoRR"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1145\/1297105.1297033"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1145\/2668930.2688820"},{"key":"ref15","first-page":"18","article-title":"Measurement, psychophysics, and utility","author":"stevens","year":"1959","journal-title":"Measurement Definitions and Theories"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1145\/1134285.1134333"},{"key":"ref16","author":"lilja","year":"2008","journal-title":"Measuring Computer Performance A Practitioner?s Guide"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2008.09.009"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1126\/science.103.2684.677"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1145\/5666.5673"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.6028\/NIST.TN.1830"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/1040136.1040137"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1145\/2442776.2442781"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4939-1905-5_4"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/1508284.1508275"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/CCGRID.2017.71"},{"key":"ref5","article-title":"Ready for rain? A view from SPEC research on the future of cloud metrics","author":"herbst","year":"2016","journal-title":"CoRR"},{"key":"ref8","article-title":"Experimental computer science: The need for a cultural change","author":"feitelson","year":"2005","journal-title":"Tech Rep"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2005.97"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1145\/2851553.2851573"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1145\/1134285.1134500"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1145\/2479871.2479892"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1080\/00031305.2015.1089789"},{"key":"ref48","article-title":"Nanotrusting the nanotime","author":"shipilev","year":"2014"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1145\/1806596.1806618"},{"key":"ref42","author":"huff","year":"1993","journal-title":"How to Lie with Statistics"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1093\/cje\/bet075"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1016\/B978-0-08-092367-3.50016-2"},{"key":"ref43","author":"reinhart","year":"2015","journal-title":"Statistics Done Wrong The Woefully Complete Guide"}],"container-title":["IEEE Transactions on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/ieeexplore.ieee.org\/ielam\/32\/9512528\/8758926-aam.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/32\/9512528\/08758926.pdf?arnumber=8758926","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,5,10]],"date-time":"2022-05-10T14:50:06Z","timestamp":1652194206000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/8758926\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,8,1]]},"references-count":83,"journal-issue":{"issue":"8"},"URL":"https:\/\/doi.org\/10.1109\/tse.2019.2927908","relation":{},"ISSN":["0098-5589","1939-3520","2326-3881"],"issn-type":[{"value":"0098-5589","type":"print"},{"value":"1939-3520","type":"electronic"},{"value":"2326-3881","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,8,1]]}}}