{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,10]],"date-time":"2026-01-10T00:39:04Z","timestamp":1768005544938,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":26,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,10,21]],"date-time":"2026-10-21T00:00:00Z","timestamp":1792540800000},"content-version":"vor","delay-in-days":449,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["CNS-2446321"],"award-info":[{"award-number":["CNS-2446321"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,7,29]]},"DOI":"10.1145\/3736731.3746151","type":"proceedings-article","created":{"date-parts":[[2025,10,21]],"date-time":"2025-10-21T11:58:19Z","timestamp":1761047899000},"page":"96-107","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Reproducibility in Applied Security Conferences: An 11-Year Review on Artifacts and Evaluation Committees"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-7807-7941","authenticated-orcid":false,"given":"Daniel","family":"Olszewski","sequence":"first","affiliation":[{"name":"University of Florida, Gainesville, FL, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-2525-4478","authenticated-orcid":false,"given":"Allison","family":"Lu","sequence":"additional","affiliation":[{"name":"University of Florida, Gainesville, FL, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-3522-4282","authenticated-orcid":false,"given":"Anna","family":"Crowder","sequence":"additional","affiliation":[{"name":"University of Florida, Gainesville, FL, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3315-938X","authenticated-orcid":false,"given":"Nathaniel","family":"Bennett","sequence":"additional","affiliation":[{"name":"University of Florida, Gainesville, FL, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-7989-7350","authenticated-orcid":false,"given":"Seth","family":"Layton","sequence":"additional","affiliation":[{"name":"University of Florida, Gainesville, FL, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-1027-5002","authenticated-orcid":false,"given":"Sri Hrushikesh","family":"Varma Bhupathiraju","sequence":"additional","affiliation":[{"name":"University of Florida, Gainesville, FL, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0672-9479","authenticated-orcid":false,"given":"Tyler","family":"Tucker","sequence":"additional","affiliation":[{"name":"University of Florida, Gainesville, FL, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-3869-9907","authenticated-orcid":false,"given":"Siddhant","family":"Kalgutkar","sequence":"additional","affiliation":[{"name":"University of Florida, Gainesville, FL, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-4052-4787","authenticated-orcid":false,"given":"Hunter","family":"Ver Helst","sequence":"additional","affiliation":[{"name":"University of Florida, Gainesville, FL, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-4717-6316","authenticated-orcid":false,"given":"Carson","family":"Stillman","sequence":"additional","affiliation":[{"name":"University of Florida, Gainesville, FL, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7498-4239","authenticated-orcid":false,"given":"Kevin R. B.","family":"Butler","sequence":"additional","affiliation":[{"name":"University of Florida, Gainesville, FL, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3630-6269","authenticated-orcid":false,"given":"Sara","family":"Rampazzi","sequence":"additional","affiliation":[{"name":"University of Florida, Gainesville, FL, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7143-5189","authenticated-orcid":false,"given":"Patrick","family":"Traynor","sequence":"additional","affiliation":[{"name":"University of Florida, Gainesville, FL, USA"}]}],"member":"320","published-online":{"date-parts":[[2025,10,21]]},"reference":[{"key":"e_1_3_3_2_2_2","unstructured":"[n. d.]. PyPDF2. https:\/\/pypi.org\/project\/PyPDF2\/."},{"key":"e_1_3_3_2_3_2","unstructured":"[n. d.]. Security and Privacy Heterogeneous Environment for Reproducible Experimentation. https:\/\/sphere-project.net\/."},{"key":"e_1_3_3_2_4_2","unstructured":"[n. d.]. Sunsetting Python 2. https:\/\/www.python.org\/doc\/sunset-python-2\/."},{"key":"e_1_3_3_2_5_2","unstructured":"2017. ACSAC Call for Artifacts. https:\/\/www.acsac.org\/2017\/artifacts\/."},{"key":"e_1_3_3_2_6_2","unstructured":"2017. WiSec Artifactiation Evaluation Committee 2017. https:\/\/wisec2017.ccs.neu.edu\/reproducibility.html."},{"key":"e_1_3_3_2_7_2","unstructured":"2020. Artifact review and badging - current. https:\/\/www.acm.org\/publications\/policies\/artifact-review-and-badging-current"},{"key":"e_1_3_3_2_8_2","unstructured":"2020. USENIX Security \u201920 Call for Artifacts. https:\/\/www.usenix.org\/conference\/usenixsecurity20\/call-for-artifacts."},{"key":"e_1_3_3_2_9_2","unstructured":"2021. USENIX Security \u201921 Call for Artifacts. https:\/\/www.usenix.org\/conference\/usenixsecurity21\/call-for-artifacts."},{"key":"e_1_3_3_2_10_2","unstructured":"2023. SIGSAC ACM CCS Call For Artifacts. https:\/\/www.sigsac.org\/ccs\/CCS2023\/call-for-artifacts.html."},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"publisher","DOI":"10.1145\/3427228.3427270"},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"crossref","unstructured":"Terry Benzel. 2023. Security and Privacy Research Artifacts: Are We Making Progress? IEEE Security & Privacy 21 01 (2023) 4\u20136.","DOI":"10.1109\/MSEC.2022.3222887"},{"key":"e_1_3_3_2_13_2","unstructured":"Christian Collberg Todd Proebsting and Alex\u00a0M Warren. 2015. Repeatability and benefaction in computer systems research. University of Arizona TR 14 4 (2015)."},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"publisher","DOI":"10.1145\/2664243.2664288"},{"key":"e_1_3_3_2_15_2","unstructured":"Jacob Devlin Ming-Wei Chang Kenton Lee and Kristina Toutanova. 2018. Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1810.04805 (2018)."},{"key":"e_1_3_3_2_16_2","doi-asserted-by":"crossref","unstructured":"Odd\u00a0Erik Gundersen. 2021. The fundamental principles of reproducibility. Philosophical Transactions of the Royal Society A 379 2197 (2021) 20200210.","DOI":"10.1098\/rsta.2020.0210"},{"key":"e_1_3_3_2_17_2","doi-asserted-by":"publisher","DOI":"10.1109\/EuroSP53844.2022.00034"},{"key":"e_1_3_3_2_18_2","unstructured":"Simon Malcomber Margaret Martonosi James\u00a0L. Moore Susan\u00a0S. Margulies Alexandra\u00a0R. Isern Alicia\u00a0J. Knoedler Kendra\u00a0V. Sharp Sean\u00a0L. Jones Kellina\u00a0M. Craig-Henderson and Erwin Gianchandani. 2022. Dear Colleague Letter: Reproducibility and Replicability in Science. (22 Oct 2022)."},{"key":"e_1_3_3_2_19_2","volume-title":"Reproducibility and replicability in science","author":"others National Academies of Sciences Engineering and Medicine and","year":"2019","unstructured":"National Academies of Sciences Engineering and Medicine and others. 2019. Reproducibility and replicability in science. National Academies Press."},{"key":"e_1_3_3_2_20_2","unstructured":"Daniel Olszewski Allison Lu Carson Stillman Kevin Warren Cole Kitroser Alejandro Pascual Divyajyoti Ukirde Kevin Butler and Patrick Traynor. [n. d.]. Artifacts for \"Get in Researchers; We\u2019re Measuring Reproducibility\". https:\/\/github.com\/reproducibility-sec\/reproducibility."},{"key":"e_1_3_3_2_21_2","doi-asserted-by":"publisher","DOI":"10.1145\/3576915.3623130"},{"key":"e_1_3_3_2_22_2","unstructured":"Alina Petukhova Joao\u00a0P Matos-Carvalho and Nuno Fachada. 2024. Text clustering with LLM embeddings. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2403.15112 (2024)."},{"key":"e_1_3_3_2_23_2","unstructured":"Edward Raff. 2019. A step toward quantifying independently reproducible machine learning research. Advances in Neural Information Processing Systems 32 (2019)."},{"key":"e_1_3_3_2_24_2","doi-asserted-by":"publisher","DOI":"10.1109\/EuroSP57164.2023.00034"},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"crossref","unstructured":"Moritz Schloegel Nils Bars Nico Schiller Lukas Bernhard Tobias Scharnowski Addison Crump Arash\u00a0Ale Ebrahim Nicolai Bissantz Marius Muench and Thorsten Holz. 2024. SoK: Prudent Evaluation Practices for Fuzzing. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2405.10220 (2024).","DOI":"10.1109\/SP54263.2024.00137"},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"publisher","DOI":"10.1145\/3427228.3427261"},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"crossref","unstructured":"Alvin Subakti Hendri Murfi and Nora Hariadi. 2022. The performance of BERT as data representation of text clustering. Journal of big Data 9 1 (2022) 15.","DOI":"10.1186\/s40537-022-00564-9"}],"event":{"name":"ACM REP '25: ACM Conference on Reproducibility and Replicability","location":"Vancouver Canada","acronym":"ACM REP '25","sponsor":["EIGREP Emerging Interest Group on Reproducibility and Replicability"]},"container-title":["Proceedings of the 3rd ACM Conference on Reproducibility and Replicability"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3736731.3746151","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3736731.3746151","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,9]],"date-time":"2026-01-09T18:04:09Z","timestamp":1767981849000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3736731.3746151"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,29]]},"references-count":26,"alternative-id":["10.1145\/3736731.3746151","10.1145\/3736731"],"URL":"https:\/\/doi.org\/10.1145\/3736731.3746151","relation":{},"subject":[],"published":{"date-parts":[[2025,7,29]]},"assertion":[{"value":"2025-10-21","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}