{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,10]],"date-time":"2026-01-10T00:47:51Z","timestamp":1768006071058,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":48,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,7,29]]},"DOI":"10.1145\/3736731.3746152","type":"proceedings-article","created":{"date-parts":[[2025,10,21]],"date-time":"2025-10-21T11:58:19Z","timestamp":1761047899000},"page":"108-120","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Lessons Learned from Five Years of Artifact Evaluations at EuroSys"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4358-976X","authenticated-orcid":false,"given":"Daniele Cono","family":"D'Elia","sequence":"first","affiliation":[{"name":"Sapienza University of Rome, Rome, Italy"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3197-839X","authenticated-orcid":false,"given":"Thaleia Dimitra","family":"Doudali","sequence":"additional","affiliation":[{"name":"IMDEA Software Institute, Madrid, Spain"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8329-5929","authenticated-orcid":false,"given":"Cristiano","family":"Giuffrida","sequence":"additional","affiliation":[{"name":"VU Amsterdam, Amsterdam, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6916-2866","authenticated-orcid":false,"given":"Miguel","family":"Matos","sequence":"additional","affiliation":[{"name":"IST Lisbon & INESC-ID, Lisbon, Portugal"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5054-7547","authenticated-orcid":false,"given":"Mathias","family":"Payer","sequence":"additional","affiliation":[{"name":"EPFL, Lausanne, Switzerland"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-4336-1316","authenticated-orcid":false,"given":"Solal","family":"Pirelli","sequence":"additional","affiliation":[{"name":"Independent Researcher, Lausanne, Switzerland"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1665-803X","authenticated-orcid":false,"given":"Georgios","family":"Portokalidis","sequence":"additional","affiliation":[{"name":"IMDEA Software Institute, Madrid, Spain"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1493-6603","authenticated-orcid":false,"given":"Valerio","family":"Schiavoni","sequence":"additional","affiliation":[{"name":"University of Neuch\u00e2tel, Neuch\u00e2tel, Switzerland"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6628-0541","authenticated-orcid":false,"given":"Salvatore","family":"Signorello","sequence":"additional","affiliation":[{"name":"NOVA University Lisbon, Lisbon, Portugal"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6983-0630","authenticated-orcid":false,"given":"Anjo","family":"Vahldiek-Oberwagner","sequence":"additional","affiliation":[{"name":"Intel Labs, Berlin, Germany"}]}],"member":"320","published-online":{"date-parts":[[2025,10,21]]},"reference":[{"key":"e_1_3_3_1_2_2","volume-title":"ACM SIGMOD Experimental Repeatability Requirements","year":"2008","unstructured":"2008. ACM SIGMOD Experimental Repeatability Requirements. http:\/\/www.sigmod08.org\/sigmod_research.shtml"},{"key":"e_1_3_3_1_3_2","volume-title":"ESEC\/FSE 2011 Call for Artifact Evaluation","year":"2011","unstructured":"2011. ESEC\/FSE 2011 Call for Artifact Evaluation. http:\/\/2011.esec-fse.org\/cfp-artifact-evaluation"},{"key":"e_1_3_3_1_4_2","volume-title":"Software Heritage","year":"2011","unstructured":"2011. Software Heritage. http:\/\/www.softwareheritage.org"},{"key":"e_1_3_3_1_5_2","volume-title":"EuroSys \u201921: Proceedings of the Sixteenth European Conference on Computer Systems","year":"2021","unstructured":"2021. EuroSys \u201921: Proceedings of the Sixteenth European Conference on Computer Systems (Online Event, United Kingdom). Association for Computing Machinery, New York, NY, USA."},{"key":"e_1_3_3_1_6_2","volume-title":"Artifact Evaluation report for EuroSys 2022","year":"2022","unstructured":"2022. Artifact Evaluation report for EuroSys 2022. https:\/\/sysartifacts.github.io\/eurosys2022\/report"},{"key":"e_1_3_3_1_7_2","volume-title":"EuroSys \u201922: Proceedings of the Seventeenth European Conference on Computer Systems","year":"2022","unstructured":"2022. EuroSys \u201922: Proceedings of the Seventeenth European Conference on Computer Systems (Rennes, France). Association for Computing Machinery, New York, NY, USA."},{"key":"e_1_3_3_1_8_2","volume-title":"EuroSys \u201923: Proceedings of the Eighteenth European Conference on Computer Systems","year":"2023","unstructured":"2023. EuroSys \u201923: Proceedings of the Eighteenth European Conference on Computer Systems (Rome, Italy). Association for Computing Machinery, New York, NY, USA."},{"key":"e_1_3_3_1_9_2","volume-title":"34th Usenix Security Symposium - Announcement and Preliminary Call for Papers.","year":"2024","unstructured":"2024. 34th Usenix Security Symposium - Announcement and Preliminary Call for Papers.https:\/\/www.usenix.org\/sites\/default\/files\/sec25_cfp_061824.pdf"},{"key":"e_1_3_3_1_10_2","volume-title":"Artifact Evaluation \u2013 ASE 2024","year":"2024","unstructured":"2024. Artifact Evaluation \u2013 ASE 2024. https:\/\/conf.researchr.org\/track\/ase-2024\/ase-2024-artifact-evaluation-track"},{"key":"e_1_3_3_1_11_2","volume-title":"EuroSys \u201924: Proceedings of the Nineteenth European Conference on Computer Systems","year":"2024","unstructured":"2024. EuroSys \u201924: Proceedings of the Nineteenth European Conference on Computer Systems (Athens, Greece). Association for Computing Machinery, New York, NY, USA."},{"key":"e_1_3_3_1_12_2","volume-title":"ACM Emerging Interest Group for Reproducibility and Replicability (EIGREP)","year":"2025","unstructured":"2025. ACM Emerging Interest Group for Reproducibility and Replicability (EIGREP). https:\/\/reproducibility.acm.org\/"},{"key":"e_1_3_3_1_13_2","volume-title":"ACM SIGMOBILE Research Papers Artifact Evaluation Guidelines","year":"2025","unstructured":"2025. ACM SIGMOBILE Research Papers Artifact Evaluation Guidelines. https:\/\/www.sigmobile.org\/grav\/about\/artifact-guidelines"},{"key":"e_1_3_3_1_14_2","volume-title":"ACM SIGMOD ARI (Availability & Reproducibility Initiative)","year":"2025","unstructured":"2025. ACM SIGMOD ARI (Availability & Reproducibility Initiative). https:\/\/reproducibility.sigmod.org\/"},{"key":"e_1_3_3_1_15_2","volume-title":"Artifact Analysis Scripts","year":"2025","unstructured":"2025. Artifact Analysis Scripts. https:\/\/github.com\/secartifacts\/artifact_analysis"},{"key":"e_1_3_3_1_16_2","volume-title":"Artifact Evaluation \u2013 CAV 2025","year":"2025","unstructured":"2025. Artifact Evaluation \u2013 CAV 2025. https:\/\/conferences.i-cav.org\/2025\/artifact\/"},{"key":"e_1_3_3_1_17_2","volume-title":"Artifact Evaluation \u2013 FSE 2025","year":"2025","unstructured":"2025. Artifact Evaluation \u2013 FSE 2025. https:\/\/conf.researchr.org\/track\/fse-2025\/fse-2025-artifacts"},{"key":"e_1_3_3_1_18_2","volume-title":"Artifact Evaluation \u2013 ICSE 2025","year":"2025","unstructured":"2025. Artifact Evaluation \u2013 ICSE 2025. https:\/\/conf.researchr.org\/track\/icse-2025\/icse-2025-artifact-evaluation"},{"key":"e_1_3_3_1_19_2","volume-title":"Dryad\u2019s website","year":"2025","unstructured":"2025. Dryad\u2019s website. https:\/\/datadryad.org\/"},{"key":"e_1_3_3_1_20_2","volume-title":"EuroSys \u201925: Proceedings of the Twentieth European Conference on Computer Systems","year":"2025","unstructured":"2025. EuroSys \u201925: Proceedings of the Twentieth European Conference on Computer Systems (Rotterdam, Netherlands). Association for Computing Machinery, New York, NY, USA."},{"key":"e_1_3_3_1_21_2","volume-title":"Figshare\u2019s website","year":"2025","unstructured":"2025. Figshare\u2019s website. https:\/\/figshare.com\/"},{"key":"e_1_3_3_1_22_2","volume-title":"Fuzzing workshop 2025","year":"2025","unstructured":"2025. Fuzzing workshop 2025. https:\/\/fuzzingworkshop.github.io\/"},{"key":"e_1_3_3_1_23_2","volume-title":"ML Reproducibility Challenge","year":"2025","unstructured":"2025. ML Reproducibility Challenge. https:\/\/reproml.org\/"},{"key":"e_1_3_3_1_24_2","volume-title":"Security Research Artifacts","year":"2025","unstructured":"2025. Security Research Artifacts. https:\/\/secartifacts.github.io\/"},{"key":"e_1_3_3_1_25_2","volume-title":"SIGSOFT Artifact Evaluation Working Group","year":"2025","unstructured":"2025. SIGSOFT Artifact Evaluation Working Group. https:\/\/github.com\/acmsigsoft\/artifact-evaluation"},{"key":"e_1_3_3_1_26_2","volume-title":"Systems Research Artifacts","year":"2025","unstructured":"2025. Systems Research Artifacts. https:\/\/sysartifacts.github.io\/"},{"key":"e_1_3_3_1_27_2","volume-title":"TACAS 2025","year":"2025","unstructured":"2025. TACAS 2025. https:\/\/etaps.org\/2025\/conferences\/tacas\/"},{"key":"e_1_3_3_1_28_2","volume-title":"Zenodo\u2019s website","year":"2025","unstructured":"2025. Zenodo\u2019s website. https:\/\/zenodo.org\/"},{"key":"e_1_3_3_1_29_2","volume-title":"Artifact Review and Badging Version 1.1","author":"Machinery Association for Computing","year":"2020","unstructured":"Association for Computing Machinery. 2020. Artifact Review and Badging Version 1.1. https:\/\/www.acm.org\/publications\/policies\/artifact-review-and-badging-current"},{"key":"e_1_3_3_1_30_2","doi-asserted-by":"publisher","unstructured":"Monya Baker. 2016. 1 500 scientists lift the lid on reproducibility. Nature 533 7604 (May 2016) 452\u2013454. 10.1038\/533452a","DOI":"10.1038\/533452a"},{"key":"e_1_3_3_1_31_2","doi-asserted-by":"publisher","unstructured":"Bachir Bendrissou Rahul Gopinath and Andreas Zeller. 2022. \u201cSynthesizing input grammars\u201d: a replication study(PLDI 2022). Association for Computing Machinery New York NY USA 260\u2013268. 10.1145\/3519939.3523716","DOI":"10.1145\/3519939.3523716"},{"key":"e_1_3_3_1_32_2","doi-asserted-by":"publisher","unstructured":"Philippe Bonnet Stefan Manegold Matias Bj\u00f8rling Wei Cao Javier Gonzalez Joel Granados Nancy Hall Stratos Idreos Milena Ivanova Ryan Johnson David Koop Tim Kraska Ren\u00e9 M\u00fcller Dan Olteanu Paolo Papotti Christine Reilly Dimitris Tsirogiannis Cong Yu Juliana Freire and Dennis Shasha. 2011. Repeatability and workability evaluation of SIGMOD 2011. SIGMOD Rec. 40 2 (Sept. 2011) 45\u201348. 10.1145\/2034863.2034873","DOI":"10.1145\/2034863.2034873"},{"key":"e_1_3_3_1_33_2","doi-asserted-by":"publisher","unstructured":"Christopher\u00a0D. Chambers and Loukia Tzavella. 2022. The past present and future of Registered Reports. Nature Human Behaviour 6 1 (Jan. 2022) 29\u201342. 10.1038\/s41562-021-01193-7","DOI":"10.1038\/s41562-021-01193-7"},{"key":"e_1_3_3_1_34_2","doi-asserted-by":"publisher","unstructured":"Andy Cockburn Pierre Dragicevic Lonni Besan\u00e7on and Carl Gutwin. 2020. Threats of a replication crisis in empirical computer science. Commun. ACM 63 8 (July 2020) 70\u201379. 10.1145\/3360311","DOI":"10.1145\/3360311"},{"key":"e_1_3_3_1_35_2","doi-asserted-by":"publisher","unstructured":"Christian Collberg and Todd\u00a0A. Proebsting. 2016. Repeatability in computer systems research. Commun. ACM 59 3 (Feb. 2016) 62\u201369. 10.1145\/2812803","DOI":"10.1145\/2812803"},{"key":"e_1_3_3_1_36_2","volume-title":"ASPLOS Artifact Evaluation badges","author":"project Enzian","year":"2022","unstructured":"Enzian project. 2022. ASPLOS Artifact Evaluation badges. https:\/\/enzian.systems\/updates\/2022\/02\/04\/Badges\/"},{"key":"e_1_3_3_1_37_2","doi-asserted-by":"publisher","unstructured":"Dror\u00a0G. Feitelson. 2015. From Repeatability to Reproducibility and Corroboration. SIGOPS Oper. Syst. Rev. 49 1 (Jan. 2015) 3\u201311. 10.1145\/2723872.2723875","DOI":"10.1145\/2723872.2723875"},{"key":"e_1_3_3_1_38_2","doi-asserted-by":"publisher","unstructured":"Quentin Guilloteau Florina Ciorba Millian Poquet Dorian Goepp and Olivier Richard. 2024. Longevity of Artifacts in Leading Parallel and Distributed Systems Conferences: a Review of the State of the Practice in 2023(ACM REP \u201924). Association for Computing Machinery New York NY USA 121\u2013133. 10.1145\/3641525.3663631","DOI":"10.1145\/3641525.3663631"},{"key":"e_1_3_3_1_39_2","doi-asserted-by":"publisher","unstructured":"Ben Hermann Stefan Winter and Janet Siegmund. 2020. Community expectations for research artifacts and evaluation processes(ESEC\/FSE 2020). Association for Computing Machinery New York NY USA 469\u2013480. 10.1145\/3368089.3409767","DOI":"10.1145\/3368089.3409767"},{"key":"e_1_3_3_1_40_2","doi-asserted-by":"publisher","unstructured":"Shriram Krishnamurthi. 2013. Artifact evaluation for software conferences. SIGPLAN Not. 48 4S (July 2013) 17\u201321. 10.1145\/2502508.2502518","DOI":"10.1145\/2502508.2502518"},{"key":"e_1_3_3_1_41_2","doi-asserted-by":"publisher","unstructured":"Tanu Malik Anjo Vahldiek-Oberwagner Ivo Jimenez and Carlos Maltzahn. 2022. Expanding the Scope of Artifact Evaluation at HPC Conferences: Experience of SC21(P-RECS \u201922). Association for Computing Machinery New York NY USA 3\u20139. 10.1145\/3526062.3536354","DOI":"10.1145\/3526062.3536354"},{"key":"e_1_3_3_1_42_2","volume-title":"Influential Reinhart-Rogoff economics paper suffers spreadsheet error","author":"Marcus Adam","year":"2013","unstructured":"Adam Marcus. 2013. Influential Reinhart-Rogoff economics paper suffers spreadsheet error. https:\/\/retractionwatch.com\/2013\/04\/18\/influential-reinhart-rogoff-economics-paper-suffers-database-error\/"},{"key":"e_1_3_3_1_43_2","doi-asserted-by":"publisher","unstructured":"Jeffrey\u00a0C. Mogul Priya Mahadevan Christophe Diot John Wilkes Phillipa Gill and Amin Vahdat. 2021. Data-driven networking research: models for academic collaboration with industry (a Google point of view). 51 4 (Dec. 2021) 47\u201349. 10.1145\/3503954.3503960","DOI":"10.1145\/3503954.3503960"},{"key":"e_1_3_3_1_44_2","doi-asserted-by":"publisher","unstructured":"National Information Standards Organization. 2021. Reproducibility Badging and Definitions. (2021). 10.3789\/niso-rp-31-2021","DOI":"10.3789\/niso-rp-31-2021"},{"key":"e_1_3_3_1_45_2","volume-title":"Artifact evaluation, present and future","author":"Pirelli Solal","year":"2022","unstructured":"Solal Pirelli. 2022. Artifact evaluation, present and future. https:\/\/www.sigops.org\/2022\/artifact-evaluation-present-and-future\/"},{"key":"e_1_3_3_1_46_2","doi-asserted-by":"publisher","unstructured":"Beth\u00a0A. Plale Tanu Malik and Line\u00a0C. Pouchard. 2021. Reproducibility Practice in High-Performance Computing: Community Survey Results. Computing in Science & Engineering 23 05 (Sept. 2021) 55\u201360. 10.1109\/MCSE.2021.3096678","DOI":"10.1109\/MCSE.2021.3096678"},{"key":"e_1_3_3_1_47_2","unstructured":"Mohammad Reza\u00a0Saleh Sedghpour Alessandro\u00a0Vittorio Papadopoulos Cristian Klein and Johan Tordsson. 2024. Artifact Evaluation for Distributed Systems: Current Practices and Beyond. arxiv:https:\/\/arXiv.org\/abs\/2406.13045\u00a0[cs.DC] https:\/\/arxiv.org\/abs\/2406.13045"},{"key":"e_1_3_3_1_48_2","volume-title":"Data Falsificada (Part 1): \u201cClusterfake\u201d\u2019","author":"Simonsohn Uri","year":"2023","unstructured":"Uri Simonsohn, Leif Nelson, and Joe Simmons. 2023. Data Falsificada (Part 1): \u201cClusterfake\u201d\u2019. https:\/\/datacolada.org\/109"},{"key":"e_1_3_3_1_49_2","unstructured":"Roberta\u00a0De Viti Solal Pirelli and Vaastav Anand. 2023. HotOS XIX Panel Report: Panel on Future of Reproduction and Replication of Systems Research. arxiv:https:\/\/arXiv.org\/abs\/2308.05762\u00a0[cs.OS] https:\/\/arxiv.org\/abs\/2308.05762"}],"event":{"name":"ACM REP '25: ACM Conference on Reproducibility and Replicability","location":"Vancouver Canada","acronym":"ACM REP '25","sponsor":["EIGREP Emerging Interest Group on Reproducibility and Replicability"]},"container-title":["Proceedings of the 3rd ACM Conference on Reproducibility and Replicability"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3736731.3746152","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,9]],"date-time":"2026-01-09T18:04:15Z","timestamp":1767981855000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3736731.3746152"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,29]]},"references-count":48,"alternative-id":["10.1145\/3736731.3746152","10.1145\/3736731"],"URL":"https:\/\/doi.org\/10.1145\/3736731.3746152","relation":{},"subject":[],"published":{"date-parts":[[2025,7,29]]},"assertion":[{"value":"2025-10-21","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}