{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:21:59Z","timestamp":1785543719376,"version":"3.56.0"},"publisher-location":"New York, NY, USA","reference-count":45,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,7,18]],"date-time":"2023-07-18T00:00:00Z","timestamp":1689638400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,7,19]]},"DOI":"10.1145\/3539618.3591931","type":"proceedings-article","created":{"date-parts":[[2023,7,19]],"date-time":"2023-07-19T00:22:59Z","timestamp":1689726179000},"page":"2629-2638","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":34,"title":["Take a Fresh Look at Recommender Systems from an Evaluation Standpoint"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0764-4258","authenticated-orcid":false,"given":"Aixin","family":"Sun","sequence":"first","affiliation":[{"name":"Nanyang Technological University, Singapore, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,7,18]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proceedings of the 5th Workshop on Online Recommender Systems and User Modeling co-located with the 16th ACM Conference on Recommender, Jo\u00e3o Vinagre, Marie Al-Ghossein, Al\u00edpio M\u00e1rio Jorge","author":"Anil Rohan","unstructured":"Rohan Anil, Sandra Gadanho, Da Huang, Nijith Jacob, Zhuoshu Li, Dong Lin, Todd Phillips, Cristina Pop, Kevin Regan, Gil I. Shamir, Rakesh Shivanna, and Qiqi Yan. 2022. On the Factory Floor: ML Engineering for Industrial-Scale Ads Recommendation Models. In Proceedings of the 5th Workshop on Online Recommender Systems and User Modeling co-located with the 16th ACM Conference on Recommender, Jo\u00e3o Vinagre, Marie Al-Ghossein, Al\u00edpio M\u00e1rio Jorge, Albert Bifet, and Ladislav Peska (Eds.), Vol. 3303. CEUR-WS.org, Seattle, WA, USA."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1002\/aaai.12051"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3137597.3137601"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2988450.2988454"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v42i3.18145"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3582900.3582905"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3460231.3474607"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3434185"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3298689.3347058"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/138859.138867"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4899-7637-6_8"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-0716-2197-4_15"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401063"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","unstructured":"Xiangnan He Lizi Liao Hanwang Zhang Liqiang Nie Xia Hu and Tat-Seng Chua. 2017. Neural Collaborative Filtering. In Proceedings of the 26th International Conference on World Wide Web (Perth Australia) (WWW '17). International World Wide Web Conferences Steering Committee Republic and Canton of Geneva CHE 173--182. https:\/\/doi.org\/10.1145\/3038912.3052569","DOI":"10.1145\/3038912.3052569"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-0716-2197-4_21"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v41i4.5312"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401233"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539813.3545124"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3569930"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2018.00035"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3490486.3538365"},{"key":"e_1_3_2_1_22_1","volume-title":"CVTT: Cross-Validation Through Time. CoRR","author":"Kolesnikov Sergey","year":"2022","unstructured":"Sergey Kolesnikov and Mikhail Andronov. 2022. CVTT: Cross-Validation Through Time. CoRR, Vol. abs\/2205.05393 (2022), 1--12."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1572133"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3523227.3548485"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371786"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.bdr.2021.100300"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","unstructured":"Weiwen Liu Yunjia Xi Jiarui Qin Fei Sun Bo Chen Weinan Zhang Rui Zhang and Ruiming Tang. 2022a. Neural Re-ranking in Multi-stage Recommender Systems: A Review. In IJCAI Luc De Raedt (Ed.). ijcai.org 5512--5520. https:\/\/doi.org\/10.24963\/ijcai.2022\/771","DOI":"10.24963\/ijcai.2022"},{"key":"e_1_3_2_1_28_1","unstructured":"Zhuoran Liu Leqi Zou Xuan Zou Caihua Wang Biao Zhang Da Tang Bolin Zhu Yijie Zhu Peng Wu Ke Wang and Youlong Cheng. 2022b. Monolith: Real Time Recommendation System with Collisionless Embedding Table. In Proceedings of the 5th Workshop on Online Recommender Systems and User Modeling co-located with the 16th ACM Conference on Recommender Systems Jo\u00e3o Vinagre Marie Al-Ghossein Al\u00edpio M\u00e1rio Jorge Albert Bifet and Ladislav Peska (Eds.) Vol. 3303. CEUR-WS.org Seattle WA USA."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3383313.3418479"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3523227.3551483"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3412774"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3523227.3546785"},{"key":"e_1_3_2_1_33_1","volume-title":"Proceedings of the Twenty-Fifth Conference on Uncertainty in Artificial Intelligence (Montreal","author":"Rendle Steffen","year":"2009","unstructured":"Steffen Rendle, Christoph Freudenthaler, Zeno Gantner, and Lars Schmidt-Thieme. 2009. BPR: Bayesian Personalized Ranking from Implicit Feedback. In Proceedings of the Twenty-Fifth Conference on Uncertainty in Artificial Intelligence (Montreal, Quebec, Canada) (UAI '09). AUAI Press, Arlington, Virginia, USA, 452--461."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","unstructured":"Francesco Ricci Lior Rokach and Bracha Shapira (Eds.). 2022. Recommender Systems Handbook (3rd Ed.). Springer US. https:\/\/doi.org\/10.1007\/978-1-0716-2197-4","DOI":"10.1007\/978-1-0716-2197-4"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2206.10848"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3383313.3412489"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1007\/s12525-022-00579-3"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2021.102609"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3460231.3478848"},{"key":"e_1_3_2_1_40_1","volume-title":"Perspectives on the Evaluation of Recommender Systems Workshop (PERSPECTIVES) at RecSys22","author":"Verachtert Robin","year":"2022","unstructured":"Robin Verachtert, Lien Michiels, and Bart Goethals. 2022. Are We Forgetting Something? Correctly Evaluate a Recommender System With an Optimal Training Window. In Perspectives on the Evaluation of Recommender Systems Workshop (PERSPECTIVES) at RecSys22. Seattle, WA, USA."},{"key":"e_1_3_2_1_41_1","volume-title":"Proceedings of the Perspectives on the Evaluation of Recommender Systems Workshop co-located RecSys (CEUR Workshop Proceedings","author":"Woolridge Daniel","year":"2021","unstructured":"Daniel Woolridge, Sean Wilner, and Madeleine Glick. 2021. Sequence or Pseudo-Sequence? An Analysis of Sequential Recommendation Datasets. In Proceedings of the Perspectives on the Evaluation of Recommender Systems Workshop co-located RecSys (CEUR Workshop Proceedings, Vol. 2955), Eva Zangerle, Christine Bauer, and Alan Said (Eds.). CEUR-WS.org."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3556536"},{"key":"e_1_3_2_1_44_1","article-title":"A Revisiting Study of Appropriate Offline Evaluation for Top-N Recommendation Algorithms. ACM","author":"Zhao Wayne Xin","year":"2022","unstructured":"Wayne Xin Zhao, Zihan Lin, Zhichao Feng, Pengfei Wang, and Ji-Rong Wen. 2022. A Revisiting Study of Appropriate Offline Evaluation for Top-N Recommendation Algorithms. ACM Trans. Inf. Syst. (May 2022).","journal-title":"Trans. Inf. Syst."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531723"}],"event":{"name":"SIGIR '23: The 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Taipei Taiwan","acronym":"SIGIR '23","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591931","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3539618.3591931","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:38:07Z","timestamp":1750178287000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591931"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7,18]]},"references-count":45,"alternative-id":["10.1145\/3539618.3591931","10.1145\/3539618"],"URL":"https:\/\/doi.org\/10.1145\/3539618.3591931","relation":{},"subject":[],"published":{"date-parts":[[2023,7,18]]},"assertion":[{"value":"2023-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}