{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,10]],"date-time":"2026-06-10T03:53:58Z","timestamp":1781063638495,"version":"3.54.1"},"reference-count":44,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"7","license":[{"start":{"date-parts":[[2021,7,1]],"date-time":"2021-07-01T00:00:00Z","timestamp":1625097600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2021,7,1]],"date-time":"2021-07-01T00:00:00Z","timestamp":1625097600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2021,7,1]],"date-time":"2021-07-01T00:00:00Z","timestamp":1625097600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"Swedish Research Council VR","award":["2018-04127"],"award-info":[{"award-number":["2018-04127"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IIEEE Trans. Software Eng."],"published-print":{"date-parts":[[2021,7,1]]},"DOI":"10.1109\/tse.2019.2925345","type":"journal-article","created":{"date-parts":[[2019,6,27]],"date-time":"2019-06-27T20:13:27Z","timestamp":1561666407000},"page":"1452-1467","source":"Crossref","is-referenced-by-count":39,"title":["What's Wrong with My Benchmark Results? Studying Bad Practices in JMH Benchmarks"],"prefix":"10.1109","volume":"47","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7084-2594","authenticated-orcid":false,"given":"Diego","family":"Costa","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0474-5718","authenticated-orcid":false,"given":"Cor-Paul","family":"Bezemer","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2777-528X","authenticated-orcid":false,"given":"Philipp","family":"Leitner","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0150-8220","authenticated-orcid":false,"given":"Artur","family":"Andrzejak","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1145\/3030207.3030226"},{"key":"ref38","article-title":"Exploring methods for evaluating group differences on the NSSE and other surveys: Are the t-test and Cohen's d indices the most appropriate choices","author":"romano","year":"2006","journal-title":"Proc Annu Meet Southern Assoc Institutional Res"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/1508244.1508275"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/LES.2017.2654160"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1145\/3030207.3030213"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1145\/3196398.3196407"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1145\/2970276.2970346"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1145\/2610384.2610393"},{"key":"ref35","article-title":"JMH samples","year":"2014"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1145\/2554850.2555022"},{"key":"ref10","article-title":"Fix 627 - Consume every object created in a benchmark to avoid DCE","author":"costa","year":"2018"},{"key":"ref40","article-title":"JUnit 5.","year":"2019"},{"key":"ref11","article-title":"Fix: Configure benchmark forks to $>=$>= 1","author":"costa","year":"2018"},{"key":"ref12","article-title":"LOG4J2–2478 return the computed variables on each benchmark to avoid DCE","author":"costa","year":"2018"},{"key":"ref13","article-title":"PUBDEV-6081: Reducing the invocation JMH level setup\/teardown to only the training","author":"costa","year":"2018"},{"key":"ref14","article-title":"Reducing the usage of JMH invocation level on setup","author":"costa","year":"2018"},{"key":"ref15","article-title":"Use blackhole objects to sink the method return in a benchmak loop (JMH)","author":"costa","year":"2018"},{"key":"ref16","article-title":"Spotjmhbugs","author":"costa","year":"2018"},{"key":"ref17","article-title":"Studying bad practices in JMH benchmarks","author":"costa","year":"2018"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1145\/2451116.2451141"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/2479871.2479892"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1145\/2491894.2464160"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/1167473.1167488"},{"key":"ref27","first-page":"853","article-title":"Benchmark precision and random initial state","author":"kalibera","year":"2005","journal-title":"Proc Int Symp Perform Evaluation Comput Telecommun Syst"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-13062-5_4"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1145\/3053600.3053638"},{"key":"ref29","first-page":"57","article-title":"Automated benchmarking of Java APIs","author":"kuperberg","year":"2010","journal-title":"Proc Softw Eng 2010 - Fachtagung des GI-Fachbereichs Softwaretechnik"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1145\/1378704.1378723"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.114.3.494"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1145\/1806596.1806647"},{"key":"ref2","article-title":"Java microbenchmark harness - (the lesser of two evils)","year":"2013"},{"key":"ref9","article-title":"Avoiding the usage of JMH invocation level on setup\/teardown","author":"costa","year":"2018"},{"key":"ref1","year":"2018"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/1297027.1297033"},{"key":"ref22","article-title":"Impact of slow page load time on website performance","author":"green","year":"2016"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1145\/2095050.2095100"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1145\/174675.177907"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/2103799.2103802"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1145\/103135.103136"},{"key":"ref23","first-page":"83","article-title":"Code layout as a source of noise in JVM performance","volume":"4","author":"gu","year":"2005","journal-title":"Studia Informatica Universalis"},{"key":"ref44","first-page":"196","author":"wilcoxon","year":"1992","journal-title":"Individual Comparisons by Ranking Methods"},{"key":"ref26","article-title":"Avoiding benchmarking pitfalls on the JVM","author":"ponge","year":"2014"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/32.888628"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1145\/2668930.2688820"}],"container-title":["IEEE Transactions on Software Engineering"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/32\/9489265\/08747433.pdf?arnumber=8747433","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,5,10]],"date-time":"2022-05-10T14:50:09Z","timestamp":1652194209000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/8747433\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,7,1]]},"references-count":44,"journal-issue":{"issue":"7"},"URL":"https:\/\/doi.org\/10.1109\/tse.2019.2925345","relation":{},"ISSN":["0098-5589","1939-3520","2326-3881"],"issn-type":[{"value":"0098-5589","type":"print"},{"value":"1939-3520","type":"electronic"},{"value":"2326-3881","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,7,1]]}}}