{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,5,30]],"date-time":"2025-05-30T05:45:38Z","timestamp":1748583938241,"version":"3.40.3"},"publisher-location":"Cham","reference-count":42,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030884178"},{"type":"electronic","value":"9783030884185"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-88418-5_9","type":"book-chapter","created":{"date-parts":[[2021,9,29]],"date-time":"2021-09-29T21:04:30Z","timestamp":1632949470000},"page":"173-193","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["My Fuzzer Beats Them All! Developing a Framework for Fair Evaluation and Comparison of Fuzzers"],"prefix":"10.1007","author":[{"given":"David","family":"Paa\u00dfen","sequence":"first","affiliation":[]},{"given":"Sebastian","family":"Surminski","sequence":"additional","affiliation":[]},{"given":"Michael","family":"Rodler","sequence":"additional","affiliation":[]},{"given":"Lucas","family":"Davi","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2021,9,30]]},"reference":[{"unstructured":"Aizatsky, M., Serebryany, K., Chang, O., Arya, A., Whittaker, M.: Announcing OSS-Fuzz: continuous fuzzing for open source software (2016)","key":"9_CR1"},{"doi-asserted-by":"crossref","unstructured":"Arcuri, A., Briand, L.: A Hitchhiker\u2019s guide to statistical tests for assessing randomized algorithms in software engineering. Softw. Test. Verif. Reliab. 24, 219\u2013250 (2014)","key":"9_CR2","DOI":"10.1002\/stvr.1486"},{"doi-asserted-by":"crossref","unstructured":"Aschermann, C., Schumilo, S., Blazytko, T., Gawlik, R., Holz, T.: REDQUEEN: fuzzing with input-to-state correspondence. In: Symposium on Network and Distributed System Security (NDSS) (2019)","key":"9_CR3","DOI":"10.14722\/ndss.2019.23371"},{"unstructured":"Benjamin, D.J., Berger, J.O., Johannesson, M., Nosek, B.A., Wagenmakers, E.J., et al.: Redefine statistical significance. Hum. Nat. Behav. 2, 6\u201310 (2017)","key":"9_CR4"},{"unstructured":"Blazytko, T., et al.: GRIMOIRE: synthesizing structure while fuzzing. In: USENIX Security Symposium (2019)","key":"9_CR5"},{"doi-asserted-by":"crossref","unstructured":"B\u00f6hme, M., Falk, B.: Fuzzing: on the exponential cost of vulnerability discovery. In: Symposium on the Foundations of Software Engineering (FSE) (2020)","key":"9_CR6","DOI":"10.1145\/3368089.3409729"},{"doi-asserted-by":"crossref","unstructured":"B\u00f6hme, M., Pham, V.T., Nguyen, M.D., Roychoudhury, A.: Directed greybox fuzzing. In: ACM Conference on Computer and Communications Security (CCS) (2017)","key":"9_CR7","DOI":"10.1145\/3133956.3134020"},{"doi-asserted-by":"crossref","unstructured":"B\u00f6hme, M., Pham, V.T., Roychoudhury, A.: Coverage-based greybox fuzzing as markov chain. In: ACM Conference on Computer and Communications Security (CCS) (2016)","key":"9_CR8","DOI":"10.1145\/2976749.2978428"},{"unstructured":"Cadar, C., Dunbar, D., Engler, D.: KLEE: unassisted and automatic generation of high-coverage tests for complex systems programs. In: USENIX Conference on Operating Systems Design and Implementation (2008)","key":"9_CR9"},{"doi-asserted-by":"crossref","unstructured":"Chen, H., et al.: Hawkeye: towards a desired directed grey-box fuzzer. In: ACM Conference on Computer and Communications Security (CCS) (2018)","key":"9_CR10","DOI":"10.1145\/3243734.3243849"},{"doi-asserted-by":"crossref","unstructured":"Chen, P., Chen, H.: Angora: efficient fuzzing by principled search. In: IEEE Symposium on Security and Privacy (S&P) (2018)","key":"9_CR11","DOI":"10.1109\/SP.2018.00046"},{"unstructured":"Chen, Y., et al.: EnFuzz: ensemble fuzzing with seed synchronization among diverse fuzzers. In: USENIX Security Symposium (2019)","key":"9_CR12"},{"doi-asserted-by":"crossref","unstructured":"Cho, M., Kim, S., Kwon, T.: Intriguer: field-level constraint solving for hybrid fuzzing. In: ACM Conference on Computer and Communications Security (CCS) (2019)","key":"9_CR13","DOI":"10.1145\/3319535.3354249"},{"doi-asserted-by":"crossref","unstructured":"Dolan-Gavitt, B., et al.: LAVA: large-scale automated vulnerability addition. In: IEEE Symposium on Security and Privacy (S&P) (2016)","key":"9_CR14","DOI":"10.1109\/SP.2016.15"},{"unstructured":"Fioraldi, A., Maier, D., Ei\u00dffeldt, H., Heuse, M.: Afl++: combining incremental steps of fuzzing research. In: USENIX Workshop on Offensive Technologies (WOOT) (2020)","key":"9_CR15"},{"doi-asserted-by":"crossref","unstructured":"Fisher, R.: On the Interpretation of $$\\chi ^2$$ from contingency tables, and the calculation of P. J. R. Stat. Soc. 85, 87\u201394 (1922)","key":"9_CR16","DOI":"10.2307\/2340521"},{"unstructured":"Fisher, R.: Statistical Methods for Research Workers. Oliver and Boyd, Edinburgh (1925)","key":"9_CR17"},{"unstructured":"Gan, S., et al.: GREYONE: data flow sensitive fuzzing. In: USENIX Security Symposium (2020)","key":"9_CR18"},{"doi-asserted-by":"crossref","unstructured":"Gan, S., et al.: CollAFL: path sensitive fuzzing. In: IEEE Symposium on Security and Privacy (S&P) (2018)","key":"9_CR19","DOI":"10.1109\/SP.2018.00040"},{"unstructured":"Google: fuzzer-test-suite (2016). https:\/\/github.com\/google\/fuzzer-test-suite\/","key":"9_CR20"},{"unstructured":"Guido, D.: Your tool works better than mine? Prove it (2016). https:\/\/blog.trailofbits.com\/2016\/08\/01\/your-tool-works-better-than-mine-prove-it\/","key":"9_CR21"},{"doi-asserted-by":"crossref","unstructured":"Hazimeh, A., Herrera, A., Payer, M.: Magma: a ground-truth fuzzing benchmark. In: Proceedings of the ACM on Measurement and Analysis of Computing Systems 4 (2020)","key":"9_CR22","DOI":"10.1145\/3428334"},{"unstructured":"Hocevar, S.: zzuf (2006). https:\/\/github.com\/samhocevar\/zzuf\/","key":"9_CR23"},{"doi-asserted-by":"crossref","unstructured":"Huang, H., Yao, P., Wu, R., Shi, Q., Zhang, C.: Pangolin: incremental hybrid fuzzing with polyhedral path abstraction. In: IEEE Symposium on Security and Privacy (S&P) (2020)","key":"9_CR24","DOI":"10.1109\/SP40000.2020.00063"},{"doi-asserted-by":"crossref","unstructured":"Inozemtseva, L., Holmes, R.: Coverage is not strongly correlated with test suite effectiveness. In: International Conference on Software Engineering (ICSE) (2014)","key":"9_CR25","DOI":"10.1145\/2568225.2568271"},{"doi-asserted-by":"crossref","unstructured":"Klees, G., Ruef, A., Cooper, B., Wei, S., Hicks, M.: Evaluating fuzz testing. In: ACM Conference on Computer and Communications Security (CCS) (2018)","key":"9_CR26","DOI":"10.1145\/3243734.3243804"},{"doi-asserted-by":"crossref","unstructured":"Lemieux, C., Sen, K.: FairFuzz: a targeted mutation strategy for increasing greybox fuzz testing coverage (2018)","key":"9_CR27","DOI":"10.1145\/3238147.3238176"},{"unstructured":"Li, Y., et al.: UNIFUZZ: a holistic and pragmatic metrics-driven platform for evaluating fuzzers. In: USENIX Security Symposium (2021)","key":"9_CR28"},{"unstructured":"Lyu, C., et al.: MOPT: optimized mutation scheduling for fuzzers. In: USENIX Security Symposium (2019)","key":"9_CR29"},{"doi-asserted-by":"crossref","unstructured":"Mann, H., Whitney, D.: On a test of whether one of two random variables is stochastically larger than the other. Ann. Math. Stat. 18 (1947)","key":"9_CR30","DOI":"10.1214\/aoms\/1177730491"},{"doi-asserted-by":"crossref","unstructured":"Metzman, J., Arya, A., Szekeres, L.: FuzzBench: Fuzzer Benchmarking as a Service (2020). https:\/\/opensource.googleblog.com\/2020\/03\/fuzzbench-fuzzer-benchmarking-as-service.html","key":"9_CR31","DOI":"10.1145\/3468264.3473932"},{"unstructured":"Paa\u00dfen, D., Surminski, S., Rodler, M., Davi, L.: Public github respository of SENF. https:\/\/github.com\/uni-due-syssec\/SENF","key":"9_CR32"},{"doi-asserted-by":"crossref","unstructured":"Pham, V.T., B\u00f6hme, M., Santosa, A.E., C\u0103ciulescu, A.R., Roychoudhury, A.: Smart greybox fuzzing. IEEE Trans. Softw. Eng. (2019)","key":"9_CR33","DOI":"10.1109\/TSE.2019.2941681"},{"key":"9_CR34","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"43","DOI":"10.1007\/978-3-662-54494-5_3","volume-title":"Fundamental Approaches to Software Engineering","author":"V-T Pham","year":"2017","unstructured":"Pham, V.-T., Khurana, S., Roy, S., Roychoudhury, A.: Bucketing failing tests via symbolic analysis. In: Huisman, M., Rubin, J. (eds.) FASE 2017. LNCS, vol. 10202, pp. 43\u201359. Springer, Heidelberg (2017). https:\/\/doi.org\/10.1007\/978-3-662-54494-5_3"},{"unstructured":"R Core Team: R: A Language and Environment for Statistical Computing. R Foundation for Statistical Computing, Vienna, Austria (2019).https:\/\/www.R-project.org\/","key":"9_CR35"},{"unstructured":"Schumilo, S., Aschermann, C., Gawlik, R., Schinzel, S., Holz, T.: kAFL: hardware-assisted feedback fuzzing for OS kernels. In: USENIX Security Symposium (2017)","key":"9_CR36"},{"doi-asserted-by":"crossref","unstructured":"van Tonder, R., Kotheimer, J., Le Goues, C.: Semantic crash bucketing. In: ACM\/IEEE International Conference on Automated Software Engineering (2018)","key":"9_CR37","DOI":"10.1145\/3238147.3238200"},{"doi-asserted-by":"crossref","unstructured":"Vargha, A., Delaney, H.D.: A critique and improvement of the CL common language effect size statistics of McGraw and Wong. J. Educ. Behav. Stat. 25, 10\u2014132 (2000)","key":"9_CR38","DOI":"10.3102\/10769986025002101"},{"doi-asserted-by":"crossref","unstructured":"Wang, Y., et al.: Not all coverage measurements are equal: fuzzing for input prioritization. In: Symposium on Network and Distributed System Security (NDSS) (2020)","key":"9_CR39","DOI":"10.14722\/ndss.2020.24422"},{"unstructured":"Yue, T., et al.: EcoFuzz: adaptive energy-saving greybox fuzzing as a variant of the adversarial multi-armed bandit. In: USENIX Security Symposium (2020)","key":"9_CR40"},{"unstructured":"Zalewski, M.: Technical \u201cwhitepaper\u201d for afl-fuzz. https:\/\/lcamtuf.coredump.cx\/afl\/technical_details.txt","key":"9_CR41"},{"doi-asserted-by":"crossref","unstructured":"Zhao, L., Duan, Y., Yin, H., Xuan, J.: Send hardest problems my way: probabilistic path prioritization for hybrid fuzzing. In: Symposium on Network and Distributed System Security (NDSS) (2019)","key":"9_CR42","DOI":"10.14722\/ndss.2019.23504"}],"container-title":["Lecture Notes in Computer Science","Computer Security \u2013 ESORICS 2021"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-88418-5_9","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,9,29]],"date-time":"2021-09-29T21:09:44Z","timestamp":1632949784000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-88418-5_9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030884178","9783030884185"],"references-count":42,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-88418-5_9","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"30 September 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ESORICS","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Symposium on Research in Computer Security","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Darmstadt","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Germany","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"4 October 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"8 October 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"26","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"esorics2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/esorics2021.athene-center.de\/index.php","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"351","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"71","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"20% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3.07","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"6.06","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"The conference was held virtually due to the COVID-19 pandemic.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}