{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T06:32:20Z","timestamp":1776061940818,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":22,"publisher":"ACM","license":[{"start":{"date-parts":[[2018,6,15]],"date-time":"2018-06-15T00:00:00Z","timestamp":1529020800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"NWO","award":["PROMIMOOC"],"award-info":[{"award-number":["PROMIMOOC"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2018,6,15]]},"DOI":"10.1145\/3209950.3209955","type":"proceedings-article","created":{"date-parts":[[2018,6,5]],"date-time":"2018-06-05T17:54:00Z","timestamp":1528221240000},"page":"1-6","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":17,"title":["Fair Benchmarking Considered Difficult"],"prefix":"10.1145","author":[{"given":"Mark","family":"Raasveldt","sequence":"first","affiliation":[{"name":"Centrum Wiskunde & Informatica (CWI), Amsterdam, The Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Pedro","family":"Holanda","sequence":"additional","affiliation":[{"name":"Centrum Wiskunde & Informatica (CWI), Amsterdam, The Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Tim","family":"Gubner","sequence":"additional","affiliation":[{"name":"Centrum Wiskunde & Informatica (CWI), Amsterdam, The Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hannes","family":"M\u00fchleisen","sequence":"additional","affiliation":[{"name":"Centrum Wiskunde & Informatica (CWI), Amsterdam, The Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2018,6,15]]},"reference":[{"key":"e_1_3_2_1_2_1","volume-title":"Technical Report","author":"Decision TPC","unstructured":"2013. TPC Benchmark H (Decision Support) Standard Specification. Technical Report. Transaction Processing Performance Council. http:\/\/www.tpc.org\/tpc_documents_current_versions\/pdf\/tpc-h_v2.17.1.pdf"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1409360.1409380"},{"key":"e_1_3_2_1_4_1","volume-title":"Performance Evaluation of Computer and Communication Systems","author":"Le Boudec Jean-Yves","unstructured":"Jean-Yves Le Boudec. 2010. Performance Evaluation of Computer and Communication Systems. EPFL Press."},{"key":"e_1_3_2_1_5_1","volume-title":"Donoho","author":"Buckheit Jonathan B.","year":"1995","unstructured":"Jonathan B. Buckheit and David L. Donoho. 1995. WaveLab and Reproducible Research. Springer New York, New York, NY, 55--81."},{"key":"e_1_3_2_1_6_1","volume-title":"Claerbou and Martin Karrenfach","author":"Jon","year":"1992","unstructured":"Jon F. Claerbou and Martin Karrenfach. 1992. Electronic Documents Give Reproducible Research a New Meaning. (1992)."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.5555\/530588"},{"key":"e_1_3_2_1_8_1","volume-title":"Vectorization and SIMD. In ADMS","author":"Gubner Tim","year":"2017","unstructured":"Tim Gubner and Peter Boncz. 2017. Exploring Query Execution Strategies for JIT, Vectorization and SIMD. In ADMS 2017."},{"key":"e_1_3_2_1_9_1","unstructured":"Gernot Heiser. 2010. Systems Benchmarking Crimes. (2010). https:\/\/www.cse.unsw.edu.au\/~gernot\/benchmarking-crimes.html"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2807591.2807644"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.5555\/1098683"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/122564.1045495"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1374780.1374791"},{"key":"e_1_3_2_1_14_1","unstructured":"I. Manolescu and Stefan Manegold. 2008. Performance Evaluation in Database Research: Principles and Experience."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1508284.1508275"},{"key":"e_1_3_2_1_16_1","volume-title":"Self-Driving Database Management Systems. In CIDR 2017, Conference on Innovative Data Systems Research. http:\/\/db.cs.cmu.edu\/papers\/2017\/p42-pavlo-cidr17","author":"Pavlo Andrew","year":"2017","unstructured":"Andrew Pavlo, Gustavo Angulo, Joy Arulraj, Haibin Lin, Jiexi Lin, Lin Ma, Prashanth Menon, Todd Mowry, Matthew Perron, Ian Quah, Siddharth Santurkar, Anthony Tomasic, Skye Toor, Dana Van Aken, Ziqi Wang, Yingjun Wu, Ran Xian, and Tieying Zhang. 2017. Self-Driving Database Management Systems. In CIDR 2017, Conference on Innovative Data Systems Research. http:\/\/db.cs.cmu.edu\/papers\/2017\/p42-pavlo-cidr17.pdf"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3124680.3124719"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.14778\/3115404.3115408"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2463676.2465306"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/125223.125262"},{"key":"e_1_3_2_1_21_1","volume-title":"Benchmarking Crimes: An Emerging Threat in Systems Security. CoRR abs\/1801.02381","author":"van der Kouwe Erik","year":"2018","unstructured":"Erik van der Kouwe, Dennis Andriesse, Herbert Bos, Cristiano Giuffrida, and Gernot Heiser. 2018. Benchmarking Crimes: An Emerging Threat in Systems Security. CoRR abs\/1801.02381 (2018). http:\/\/arxiv.org\/abs\/1801.02381"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1080\/00031305.1984.10483186"},{"key":"e_1_3_2_1_23_1","unstructured":"David Wheeler. 2017. The DeWitt clause's censorship should be illegal. (2017). https:\/\/www.dwheeler.com\/essays\/dewitt-clause.html"}],"event":{"name":"SIGMOD\/PODS '18: International Conference on Management of Data","location":"Houston TX USA","acronym":"SIGMOD\/PODS '18","sponsor":["SIGMOD ACM Special Interest Group on Management of Data"]},"container-title":["Proceedings of the Workshop on Testing Database Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3209950.3209955","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3209950.3209955","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T01:39:18Z","timestamp":1750210758000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3209950.3209955"}},"subtitle":["Common Pitfalls In Database Performance Testing"],"short-title":[],"issued":{"date-parts":[[2018,6,15]]},"references-count":22,"alternative-id":["10.1145\/3209950.3209955","10.1145\/3209950"],"URL":"https:\/\/doi.org\/10.1145\/3209950.3209955","relation":{},"subject":[],"published":{"date-parts":[[2018,6,15]]},"assertion":[{"value":"2018-06-15","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}