{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,10]],"date-time":"2026-04-10T10:05:50Z","timestamp":1775815550584,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":18,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,9,14]],"date-time":"2023-09-14T00:00:00Z","timestamp":1694649600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,9,14]]},"DOI":"10.1145\/3604915.3609495","type":"proceedings-article","created":{"date-parts":[[2023,9,14]],"date-time":"2023-09-14T22:40:23Z","timestamp":1694731223000},"page":"1284-1285","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["On Challenges of Evaluating Recommender Systems in an Offline Setting"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0764-4258","authenticated-orcid":false,"given":"Aixin","family":"Sun","sequence":"first","affiliation":[{"name":"Nanyang Technological University, Singapore"}]}],"member":"320","published-online":{"date-parts":[[2023,9,14]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3527449"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1002\/aaai.12051"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3137597.3137601"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v42i3.18145"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"crossref","unstructured":"Maurizio\u00a0Ferrari Dacrema Paolo Cremonesi and Dietmar Jannach. 2019. Are we really making much progress? A worrying analysis of recent neural recommendation approaches. In RecSys. ACM 101\u2013109.","DOI":"10.1145\/3298689.3347058"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3434185"},{"key":"e_1_3_2_1_7_1","unstructured":"Yitong Ji Aixin Sun Jie Zhang and Chenliang Li. 2020. A Re-visit of the Popularity Baseline in Recommender Systems. In SIGIR. ACM 1749\u20131752."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539813.3545124"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3569930"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"crossref","unstructured":"Zaiqiao Meng Richard McCreadie Craig Macdonald and Iadh Ounis. 2020. Exploring Data Splitting Strategies for the Evaluation of Recommendation Models. In RecSys. ACM 681\u2013686.","DOI":"10.1145\/3383313.3418479"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","unstructured":"Aixin Sun. 2023. Take a Fresh Look at Recommender Systems from an Evaluation Standpoint. In SIGIR. ACM 2629\u20132638. https:\/\/doi.org\/10.1145\/3539618.3591931","DOI":"10.1145\/3539618.3591931"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2022.3231891"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","unstructured":"Zhu Sun Di Yu Hui Fang Jie Yang Xinghua Qu Jie Zhang and Cong Geng. 2020. Are We Evaluating Rigorously? Benchmarking Recommendation for Reproducible Evaluation and Fair Comparison. In RecSys. ACM 23\u201332. https:\/\/doi.org\/10.1145\/3383313.3412489","DOI":"10.1145\/3383313.3412489"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","unstructured":"Yan-Martin Tamm Rinchin Damdinov and Alexey Vasilev. 2021. Quality Metrics in Recommender Systems: Do We Calculate Metrics Consistently?. In RecSys (Amsterdam Netherlands). ACM 708\u2013713. https:\/\/doi.org\/10.1145\/3460231.3478848","DOI":"10.1145\/3460231.3478848"},{"key":"e_1_3_2_1_15_1","volume-title":"Perspectives on the Evaluation of Recommender Systems Workshop (PERSPECTIVES) at RecSys22","author":"Verachtert Robin","year":"2022","unstructured":"Robin Verachtert, Lien Michiels, and Bart Goethals. 2022. Are We Forgetting Something? Correctly Evaluate a Recommender System With an Optimal Training Window. In Perspectives on the Evaluation of Recommender Systems Workshop (PERSPECTIVES) at RecSys22. Seattle, WA, USA."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3556536"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545796"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531723"}],"event":{"name":"RecSys '23: Seventeenth ACM Conference on Recommender Systems","location":"Singapore Singapore","acronym":"RecSys '23","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGAI ACM Special Interest Group on Artificial Intelligence","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data","SIGIR ACM Special Interest Group on Information Retrieval","SIGCHI ACM Special Interest Group on Computer-Human Interaction","SIGecom Special Interest Group on Economics and Computation"]},"container-title":["Proceedings of the 17th ACM Conference on Recommender Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3604915.3609495","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3604915.3609495","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:46:35Z","timestamp":1750178795000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3604915.3609495"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,9,14]]},"references-count":18,"alternative-id":["10.1145\/3604915.3609495","10.1145\/3604915"],"URL":"https:\/\/doi.org\/10.1145\/3604915.3609495","relation":{},"subject":[],"published":{"date-parts":[[2023,9,14]]},"assertion":[{"value":"2023-09-14","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}