{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,18]],"date-time":"2026-04-18T03:15:42Z","timestamp":1776482142848,"version":"3.51.2"},"publisher-location":"New York, NY, USA","reference-count":68,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,5,21]],"date-time":"2022-05-21T00:00:00Z","timestamp":1653091200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,5,21]]},"DOI":"10.1145\/3510003.3510230","type":"proceedings-article","created":{"date-parts":[[2022,7,5]],"date-time":"2022-07-05T22:42:59Z","timestamp":1657060979000},"page":"1621-1633","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":80,"title":["On the reliability of coverage-based fuzzer benchmarking"],"prefix":"10.1145","author":[{"given":"Marcel","family":"B\u00f6hme","sequence":"first","affiliation":[{"name":"Monash University, Australia"}]},{"given":"L\u00e1szl\u00f3","family":"Szekeres","sequence":"additional","affiliation":[{"name":"Google"}]},{"given":"Jonathan","family":"Metzman","sequence":"additional","affiliation":[{"name":"Google"}]}],"member":"320","published-online":{"date-parts":[[2022,7,5]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1002\/stvr.1486"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2020.3016773"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409748"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2017.2785841"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2610384.2628058"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3106237.3106255"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3433210.3453096"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2017.61"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416667"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00082"},{"key":"e_1_3_2_1_11_1","unstructured":"LLVM Developers. 2021. Clang 13 documentation - UndefinedBehaviorSanitizer (UBSan). https:\/\/clang.llvm.org\/docs\/UndefinedBehaviorSanitizer.html. [Online; accessed 16.Aug.21]."},{"key":"e_1_3_2_1_12_1","unstructured":"LLVM Developers. 2022. Clang 15 documentation - Source-based Code Coverage. https:\/\/clang.llvm.org\/docs\/SourceBasedCodeCoverage.html#interpreting-reports. [Online; accessed 10.Feb.22]."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2016.15"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/1181775.1181787"},{"key":"e_1_3_2_1_15_1","volume-title":"14th USENIX Workshop on Offensive Technologies (WOOT 20)","author":"Fioraldi Andrea","year":"2020","unstructured":"Andrea Fioraldi, Dominik Maier, Heiko Ei\u00dffeldt, and Marc Heuse. 2020. AFL++ : Combining Incremental Steps of Fuzzing Research. In 14th USENIX Workshop on Offensive Technologies (WOOT 20). USENIX Association. https:\/\/www.usenix.org\/conference\/woot20\/presentation\/fioraldi"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.5555\/2337223.2337245"},{"key":"e_1_3_2_1_17_1","volume-title":"GREYONE: Data Flow Sensitive Fuzzing. In 29th USENIX Security Symposium (USENIX Security 20)","author":"Gan Shuitao","year":"2020","unstructured":"Shuitao Gan, Chao Zhang, Peng Chen, Bodong Zhao, Xiaojun Qin, Dong Wu, and Zuoning Chen. 2020. GREYONE: Data Flow Sensitive Fuzzing. In 29th USENIX Security Symposium (USENIX Security 20). USENIX Association, 2577--2594."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2018.00040"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/QRS51102.2020.00071"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-66299-2_5"},{"key":"e_1_3_2_1_21_1","volume-title":"An Empirical Study on Benchmarks of Artificial Software Vulnerabilities. (March","author":"Geng Sijia","year":"2020","unstructured":"Sijia Geng, Yuekang Li, Yunlan Du, Jun Xu, Yang Liu, and Bing Mao. 2020. An Empirical Study on Benchmarks of Artificial Software Vulnerabilities. (March 2020). https:\/\/arxiv.org\/abs\/2003.09561v1"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2660767"},{"key":"e_1_3_2_1_23_1","unstructured":"Google. 2021. ClusterFuzz. https:\/\/google.github.io\/clusterfuzz\/. [Online; accessed 16.Aug.21]."},{"key":"e_1_3_2_1_24_1","unstructured":"Google. 2021. OSS-Fuzz: Continuous Fuzzing for Open Source Software. https:\/\/github.com\/google\/oss-fuzz. [Online; accessed 16.Aug.21]."},{"key":"e_1_3_2_1_25_1","volume-title":"Mutation Analysis: Answering the Fuzzing Challenge. arXiv:2201.11303 [cs.SE]","author":"Gopinath Rahul","year":"2022","unstructured":"Rahul Gopinath, Philipp G\u00f6rz, and Alex Groce. 2022. Mutation Analysis: Answering the Fuzzing Challenge. arXiv:2201.11303 [cs.SE]"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2568225.2568278"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2019.00019"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3428334"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409767"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3460319.3464795"},{"key":"e_1_3_2_1_31_1","unstructured":"Marc Heuse. 2020. Twitter. https:\/\/twitter.com\/hackerschoice\/status\/1302514351811842056. [Online; accessed 16.Aug.21]."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2568225.2568271"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3340459"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2610384.2628055"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/2635868.2635929"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3243734.3243804"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1109\/SANER.2015.7081877"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238176"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510454.3516847"},{"key":"e_1_3_2_1_40_1","volume-title":"Proceedings of the 28th USENIX Conference on Security Symposium","author":"Lyu Chenyang","year":"2019","unstructured":"Chenyang Lyu, Shouling Ji, Chao Zhang, Yuwei Li, Wei-Han Lee, Yu Song, and Raheem Beyah. 2019. MOPT: Optimized Mutation Scheduling for Fuzzers. In Proceedings of the 28th USENIX Conference on Security Symposium (Santa Clara, CA, USA) (SEC'19). USENIX Association, USA, 1949--1966."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0140-6736(86)90837-8"},{"key":"e_1_3_2_1_42_1","volume-title":"Interrater reliability: the kappa statistic. Biochemia medica 22, 3","author":"McHugh Mary L","year":"2012","unstructured":"Mary L McHugh. 2012. Interrater reliability: the kappa statistic. Biochemia medica 22, 3 (2012), 276--282."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1002\/stvr.294"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW.2011.100"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3473932"},{"key":"e_1_3_2_1_46_1","unstructured":"Microsoft. 2021. OneFuzz: A self-hosted Fuzzing-As-A-Service platform. https:\/\/github.com\/microsoft\/onefuzz. [Online; accessed 16.Aug.21]."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/1315245.1315311"},{"key":"e_1_3_2_1_48_1","unstructured":"Tyler Nighswander. 2020. Twitter. https:\/\/twitter.com\/tylerni7\/status\/1374519171413766145. [Online; accessed 16.Aug.21]."},{"key":"e_1_3_2_1_49_1","volume-title":"ParmeSan: Sanitizer-guided Greybox Fuzzing. In 29th USENIX Security Symposium (USENIX Security 20)","author":"\u00d6sterlund Sebastian","year":"2020","unstructured":"Sebastian \u00d6sterlund, Kaveh Razavi, Herbert Bos, and Cristiano Giuffrida. 2020. ParmeSan: Sanitizer-guided Greybox Fuzzing. In 29th USENIX Security Symposium (USENIX Security 20). USENIX Association, 2289--2306."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180183"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468623"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/2991079.2991103"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2941681"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3236084"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1213\/ANE.0000000000002864"},{"key":"e_1_3_2_1_56_1","unstructured":"Kostya Serebryany. 2021. libFuzzer - a library for coverage-guided fuzz testing. https:\/\/llvm.org\/docs\/LibFuzzer.html. [Online; accessed 16.Aug.21]."},{"key":"e_1_3_2_1_57_1","volume-title":"Proceedings of the 2012 USENIX Conference on Annual Technical Conference (Boston, MA) (USENIX ATC'12). USENIX Association, USA, 28","author":"Serebryany Konstantin","year":"2012","unstructured":"Konstantin Serebryany, Derek Bruening, Alexander Potapenko, and Dmitry Vyukov. 2012. AddressSanitizer: A Fast Address Sanity Checker. In Proceedings of the 2012 USENIX Conference on Annual Technical Conference (Boston, MA) (USENIX ATC'12). USENIX Association, USA, 28."},{"key":"e_1_3_2_1_58_1","unstructured":"Robert Swiecki. 2021. Honggfuzz. https:\/\/github.com\/google\/honggfuzz. [Online; accessed 16.Aug.21]."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1016\/B978-012691360-6\/50005-7"},{"key":"e_1_3_2_1_60_1","volume-title":"BugSwarm: mining and continuously growing a dataset of reproducible failures and fixes","author":"Tomassi David A.","unstructured":"David A. Tomassi, Naji Dmeiri, Yichen Wang, Antara Bhowmick, Yen-Chuan Liu, Premkumar T. Devanbu, Bogdan Vasilescu, and Cindy Rubio-Gonz\u00e1lez. 2019. BugSwarm: mining and continuously growing a dataset of reproducible failures and fixes. In ICSE. IEEE \/ ACM, 339--349."},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-25231-0_5"},{"key":"e_1_3_2_1_62_1","unstructured":"Cheng Wen. 2021. Recent Papers Related To Fuzzing. https:\/\/github.com\/wcventure\/FuzzingPaper. [Online; accessed 16.Aug.21]."},{"key":"e_1_3_2_1_63_1","unstructured":"Micha\u0142Zalewski. 2021. american fuzzy lop (2.52b). https:\/\/lcamtuf.coredump.cx\/afl\/. [Online; accessed 16.Aug.21]."},{"key":"e_1_3_2_1_64_1","unstructured":"Andreas Zeller. 2022. A Tweet. https:\/\/twitter.com\/AndreasZeller\/status\/1468142858553200644. [Online; accessed 10.Feb.22]."},{"key":"e_1_3_2_1_65_1","unstructured":"Andreas Zeller Sascha Just and Kai Greshake. 2019. When Results Are All That Matters: Consequences. https:\/\/andreas-zeller.blogspot.com\/2019\/10\/when-results-are-all-that-matters.html. [Online; accessed 16.Aug.21]."},{"key":"e_1_3_2_1_66_1","unstructured":"Andreas Zeller Sascha Just and Kai Greshake. 2019. When Results Are All That Matters: The Case of the Angora Fuzzer. https:\/\/andreas-zeller.blogspot.com\/2019\/10\/when-results-are-all-that-matters-case.html. [Online; accessed 16.Aug.21]."},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/2786805.2786858"},{"key":"e_1_3_2_1_68_1","volume-title":"DeFuzz: Deep Learning Guided Directed Fuzzing. CoRR abs\/2010.12149","author":"Zhu Xiaogang","year":"2020","unstructured":"Xiaogang Zhu, Shigang Liu, Xian Li, Sheng Wen, Jun Zhang, Seyit Ahmet \u00c7amtepe, and Yang Xiang. 2020. DeFuzz: Deep Learning Guided Directed Fuzzing. CoRR abs\/2010.12149 (2020)."}],"event":{"name":"ICSE '22: 44th International Conference on Software Engineering","location":"Pittsburgh Pennsylvania","acronym":"ICSE '22","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 44th International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3510003.3510230","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3510003.3510230","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:12:24Z","timestamp":1750191144000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3510003.3510230"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,5,21]]},"references-count":68,"alternative-id":["10.1145\/3510003.3510230","10.1145\/3510003"],"URL":"https:\/\/doi.org\/10.1145\/3510003.3510230","relation":{},"subject":[],"published":{"date-parts":[[2022,5,21]]},"assertion":[{"value":"2022-07-05","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}