{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,16]],"date-time":"2026-03-16T19:00:34Z","timestamp":1773687634415,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":51,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,9,22]]},"DOI":"10.1145\/3705328.3748157","type":"proceedings-article","created":{"date-parts":[[2025,9,6]],"date-time":"2025-09-06T10:48:44Z","timestamp":1757155724000},"page":"884-893","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["\u201cWe Share Our Code Online\u201d: Why This Is Not Enough to Ensure Reproducibility and Progress in Recommender Systems Research"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6239-8198","authenticated-orcid":false,"given":"Faisal","family":"Shehzad","sequence":"first","affiliation":[{"name":"University of Klagenfurt, Klagenfurt am W\u00f6rthersee, Austria"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1765-2449","authenticated-orcid":false,"given":"Timo","family":"Breuer","sequence":"additional","affiliation":[{"name":"TH K\u00f6ln (University of Applied Sciences), K\u00f6ln, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7001-4817","authenticated-orcid":false,"given":"Maria","family":"Maistro","sequence":"additional","affiliation":[{"name":"University of Copenhagen, Copenhagen, Denmark"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4698-8507","authenticated-orcid":false,"given":"Dietmar","family":"Jannach","sequence":"additional","affiliation":[{"name":"University of Klagenfurt, Klagenfurt am W\u00f6rthersee, Austria"}]}],"member":"320","published-online":{"date-parts":[[2025,9,7]]},"reference":[{"key":"e_1_3_3_2_2_2","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463245"},{"key":"e_1_3_3_2_3_2","doi-asserted-by":"publisher","DOI":"10.1145\/3604915.3609489"},{"key":"e_1_3_3_2_4_2","doi-asserted-by":"publisher","DOI":"10.1145\/3687998"},{"key":"e_1_3_3_2_5_2","doi-asserted-by":"publisher","unstructured":"Christine Bauer Eva Zangerle and Alan Said. 2024. Exploring the Landscape of Recommender Systems Evaluation: Practices and Perspectives. Trans. Recomm. Syst. 2 1 (2024) 11:1\u201311:31. 10.1145\/3629170","DOI":"10.1145\/3629170"},{"key":"e_1_3_3_2_6_2","doi-asserted-by":"publisher","unstructured":"Joeran Beel Corinna Breitinger Stefn Langer Andreas Lommatzsch and Bela Gipp. 2016. Towards Reproducibility in Recommender-systems Research. User Model. User Adapt. Interact. 26 1 (2016) 69\u2013101. 10.1007\/s11257-016-9174-x","DOI":"10.1007\/s11257-016-9174-x"},{"key":"e_1_3_3_2_7_2","volume-title":"Dagstuhl Seminar 24211: Evaluation Perspectives of Recommender Systems: Driving Research and Education","author":"Beel Joeran","year":"2024","unstructured":"Joeran Beel, Dietmar Jannach, Alan Said, Guy Shani, Tobias Vente, and Lukas Wegmeth. 2024. Best-Practices for Offline Evaluations of Recommender Systems. In Dagstuhl Seminar 24211: Evaluation Perspectives of Recommender Systems: Driving Research and Education."},{"key":"e_1_3_3_2_8_2","doi-asserted-by":"publisher","DOI":"10.1145\/2507157.2508006"},{"key":"e_1_3_3_2_9_2","doi-asserted-by":"publisher","unstructured":"Alejandro Bellog\u00edn and Alan Said. 2021. Improving Accountability in Recommender Systems Research through Reproducibility. User Model. User Adapt. Interact. 31 5 (2021) 941\u2013977. 10.1007\/s11257-021-09302-x","DOI":"10.1007\/s11257-021-09302-x"},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"publisher","unstructured":"Carl Boettiger. 2015. An Introduction to Docker for Reproducible Research. SIGOPS Oper. Syst. Rev. 49 1 (Jan. 2015) 71\u201379. 10.1145\/2723872.2723882","DOI":"10.1145\/2723872.2723882"},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"publisher","unstructured":"Emanuele Cavenaghi Gabriele Sottocornola Fabio Stella and Markus Zanker. 2023. A Systematic Study on Reproducibility of Reinforcement Learning in Recommendation Systems. ACM Trans. Recomm. Syst. 1 3 (2023). 10.1145\/3596519","DOI":"10.1145\/3596519"},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"publisher","DOI":"10.1145\/3383313.3418486"},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"publisher","unstructured":"Paolo Cremonesi and Dietmar Jannach. 2021. Progress in Recommender Systems Research: Crisis? What Crisis? AI Mag. 42 3 (2021) 43\u201354. 10.1609\/aimag.v42i3.18145","DOI":"10.1609\/aimag.v42i3.18145"},{"key":"e_1_3_3_2_14_2","unstructured":"Maurizio\u00a0Ferrari Dacrema Michael Benigni and Nicola Ferro. 2025. Reproducibility and Artifact Consistency of the SIGIR 2022 Recommender Systems Papers Based on Message Passing. (2025). arxiv:https:\/\/arXiv.org\/abs\/2503.07823\u00a0[cs.IR] https:\/\/arxiv.org\/abs\/2503.07823"},{"key":"e_1_3_3_2_15_2","doi-asserted-by":"publisher","unstructured":"Maurizio\u00a0Ferrari Dacrema Simone Boglio Paolo Cremonesi and Dietmar Jannach. 2021. A Troubling Analysis of Reproducibility and Progress in Recommender Systems Research. ACM Trans. Inf. Syst. 39 2 (2021) 20:1\u201320:49. 10.1145\/3434185","DOI":"10.1145\/3434185"},{"key":"e_1_3_3_2_16_2","doi-asserted-by":"publisher","DOI":"10.1145\/2043932.2043958"},{"key":"e_1_3_3_2_17_2","doi-asserted-by":"publisher","DOI":"10.1145\/3298689.3347058"},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","unstructured":"Nicola Ferro and Diane Kelly. 2018. SIGIR Initiative to Implement ACM Artifact Review and Badging. SIGIR Forum 52 1 (Aug. 2018) 4\u201310. 10.1145\/3274784.3274786","DOI":"10.1145\/3274784.3274786"},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"publisher","DOI":"10.1145\/3539618.3591888"},{"key":"e_1_3_3_2_20_2","doi-asserted-by":"publisher","DOI":"10.1145\/2043932.2043989"},{"key":"e_1_3_3_2_21_2","doi-asserted-by":"publisher","unstructured":"Odd\u00a0Erik Gundersen Malte Helmert and Holger Hoos. 2024. Improving Reproducibility in AI Research: Four Mechanisms Adopted by JAIR. Journal of Artificial Intelligence Research 81 (2024) 1019\u20131041. 10.1613\/jair.1.16905","DOI":"10.1613\/jair.1.16905"},{"key":"e_1_3_3_2_22_2","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11503"},{"key":"e_1_3_3_2_23_2","doi-asserted-by":"publisher","DOI":"10.1145\/3604915.3609487"},{"key":"e_1_3_3_2_24_2","doi-asserted-by":"publisher","DOI":"10.1145\/3604915.3608839"},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","unstructured":"Frank Hopfgartner Allan Hanbury Henning M\u00fcller Ivan Eggel Krisztian Balog Torben Brodt Gordon\u00a0V. Cormack Jimmy Lin Jayashree Kalpathy-Cramer Noriko Kando Makoto\u00a0P. Kato Anastasia Krithara Tim Gollub Martin Potthast Evelyne Viegas and Simon Mercer. 2018. Evaluation-as-a-Service for the Computational Sciences: Overview and Outlook. ACM J. Data Inf. Qual. 10 4 (2018) 15:1\u201315:32. 10.1145\/3239570","DOI":"10.1145\/3239570"},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"publisher","unstructured":"Yitong Ji Aixin Sun Jie Zhang and Chenliang Li. 2023. A Critical Study on Data Leakage in Recommender System Offline Evaluation. ACM Trans. Inf. Syst. 41 3 Article 75 (Feb. 2023) 27\u00a0pages. 10.1145\/3569930","DOI":"10.1145\/3569930"},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/2532508.2532513"},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","unstructured":"Yehuda Koren Robert Bell and Chris Volinsky. 2009. Matrix Factorization Techniques for Recommender Systems. Computer 42 8 (2009) 30\u201337. 10.1109\/MC.2009.263","DOI":"10.1109\/MC.2009.263"},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2021\/651"},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"publisher","unstructured":"Walid Krichene and Steffen Rendle. 2022. On Sampled Metrics for Item Recommendation. Commun. ACM 65 7 (2022) 75\u201383. 10.1145\/3535335","DOI":"10.1145\/3535335"},{"key":"e_1_3_3_2_31_2","doi-asserted-by":"publisher","unstructured":"Sara Latifi Dietmar Jannach and Andr\u00e9s Ferraro. 2022. Sequential Recommendation: A Study on Transformers Nearest Neighbors and Sampled Metrics. Information Sciences 609 (2022) 660\u2013678. 10.1016\/j.ins.2022.07.079","DOI":"10.1016\/j.ins.2022.07.079"},{"key":"e_1_3_3_2_32_2","doi-asserted-by":"publisher","unstructured":"Sara Latifi Noemi Mauro and Dietmar Jannach. 2021. Session-aware Recommendation: A Surprising Quest for the State-of-the-art. Information Sciences 573 (2021) 291\u2013315. 10.1016\/j.ins.2021.05.048","DOI":"10.1016\/j.ins.2021.05.048"},{"key":"e_1_3_3_2_33_2","doi-asserted-by":"publisher","unstructured":"Chao Liu Cuiyun Gao Xin Xia David Lo John Grundy and Xiaohu Yang. 2021. On the Reproducibility and Replicability of Deep Learning in Software Engineering. ACM Trans. Softw. Eng. Methodol. 31 1 (2021) 46\u00a0pages. 10.1145\/3477535","DOI":"10.1145\/3477535"},{"key":"e_1_3_3_2_34_2","doi-asserted-by":"publisher","unstructured":"Malte Ludewig Sara Latifi Noemi Mauro and Dietmar Jannach. 2021. Empirical Analysis of Session-Based Recommendation Algorithms. User Modeling and User-Adapted Interaction 31 1 (2021) 149\u2013181. 10.1007\/s11257-020-09277-1","DOI":"10.1007\/s11257-020-09277-1"},{"key":"e_1_3_3_2_35_2","doi-asserted-by":"publisher","DOI":"10.1145\/3383313.3418479"},{"key":"e_1_3_3_2_36_2","doi-asserted-by":"publisher","DOI":"10.1145\/3640457.3688073"},{"key":"e_1_3_3_2_37_2","doi-asserted-by":"publisher","DOI":"10.1109\/QRS62785.2024.00038"},{"key":"e_1_3_3_2_38_2","unstructured":"David Picard. 2021. Torch.manual_seed(3407) is All You Need: On the Influence of Random Seeds in Deep Learning Architectures for Computer Vision. CoRR abs\/2109.08203 (2021). arXiv:https:\/\/arXiv.org\/abs\/2109.08203https:\/\/arxiv.org\/abs\/2109.08203"},{"key":"e_1_3_3_2_39_2","doi-asserted-by":"publisher","unstructured":"Martin Potthast Tim Gollub Matti Wiegmann and Benno Stein. 2019. TIRA Integrated Research Architecture. Information Retrieval Evaluation in a Changing World: Lessons Learned from 20 Years of CLEF (2019) 123\u2013160. 10.1007\/978-3-030-22948-15","DOI":"10.1007\/978-3-030-22948-15"},{"key":"e_1_3_3_2_40_2","doi-asserted-by":"publisher","DOI":"10.1145\/3383313.3412488"},{"key":"e_1_3_3_2_41_2","doi-asserted-by":"publisher","DOI":"10.1145\/3523227.3548486"},{"key":"e_1_3_3_2_42_2","doi-asserted-by":"publisher","DOI":"10.1145\/2645710.2645746"},{"key":"e_1_3_3_2_43_2","doi-asserted-by":"publisher","DOI":"10.1145\/2792838.2792841"},{"key":"e_1_3_3_2_44_2","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911492"},{"key":"e_1_3_3_2_45_2","doi-asserted-by":"publisher","unstructured":"Harald Semmelrock Tony Ross-Hellauer Simone Kopeinik Dieter Theiler Armin Haberl Stefan Thalmann and Dominik Kowald. 2025. Reproducibility in Machine-learning-Based Research: Overview Barriers and Drivers. AI Magazine 46 2 (2025) e70002. 10.1002\/aaai.70002","DOI":"10.1002\/aaai.70002"},{"key":"e_1_3_3_2_46_2","doi-asserted-by":"publisher","DOI":"10.1145\/3604915.3609488"},{"key":"e_1_3_3_2_47_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-56066-812"},{"key":"e_1_3_3_2_48_2","doi-asserted-by":"publisher","DOI":"10.1145\/3539618.3591931"},{"key":"e_1_3_3_2_49_2","doi-asserted-by":"publisher","DOI":"10.1145\/3383313.3412489"},{"key":"e_1_3_3_2_50_2","doi-asserted-by":"publisher","DOI":"10.1145\/3640457.3688074"},{"key":"e_1_3_3_2_51_2","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401131"},{"key":"e_1_3_3_2_52_2","doi-asserted-by":"publisher","DOI":"10.1145\/3459637.3482016"}],"event":{"name":"RecSys '25: Nineteenth ACM Conference on Recommender Systems","location":"Prague Czech Republic","acronym":"RecSys '25","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction","SIGAI ACM Special Interest Group on Artificial Intelligence","SIGIR ACM Special Interest Group on Information Retrieval","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data","SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"]},"container-title":["Proceedings of the Nineteenth ACM Conference on Recommender Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3705328.3748157","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,6]],"date-time":"2025-09-06T11:50:28Z","timestamp":1757159428000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3705328.3748157"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,9,7]]},"references-count":51,"alternative-id":["10.1145\/3705328.3748157","10.1145\/3705328"],"URL":"https:\/\/doi.org\/10.1145\/3705328.3748157","relation":{},"subject":[],"published":{"date-parts":[[2025,9,7]]},"assertion":[{"value":"2025-09-07","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}