{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,11]],"date-time":"2025-09-11T19:11:45Z","timestamp":1757617905719,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":53,"publisher":"ACM","funder":[{"DOI":"10.13039\/501100003593","name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico","doi-asserted-by":"publisher","award":[""],"award-info":[{"award-number":[""]}],"id":[{"id":"10.13039\/501100003593","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004901","name":"Funda\u00e7\u00e3o de Amparo \u00e0 Pesquisa do Estado de Minas Gerais","doi-asserted-by":"publisher","award":[""],"award-info":[{"award-number":[""]}],"id":[{"id":"10.13039\/501100004901","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,9,22]]},"DOI":"10.1145\/3705328.3748086","type":"proceedings-article","created":{"date-parts":[[2025,9,6]],"date-time":"2025-09-06T10:46:13Z","timestamp":1757155573000},"page":"360-369","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["On the Reliability of Sampling Strategies in Offline Recommender Evaluation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0004-2857-2913","authenticated-orcid":false,"given":"Bruno L.","family":"Pereira","sequence":"first","affiliation":[{"name":"Universidade Federal de Minas Gerais, Belo Horizonte, MG, Brazil"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2929-0529","authenticated-orcid":false,"given":"Alan","family":"Said","sequence":"additional","affiliation":[{"name":"University of Gothenburg, Gothenburg, Sweden"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2921-8444","authenticated-orcid":false,"given":"Rodrygo L. T.","family":"Santos","sequence":"additional","affiliation":[{"name":"Universidade Federal de Minas Gerais, Belo Horizonte, MG, Brazil"}]}],"member":"320","published-online":{"date-parts":[[2025,9,7]]},"reference":[{"key":"e_1_3_3_2_2_2","unstructured":"Himan Abdollahpouri Masoud Mansoury Robin Burke and Bamshad Mobasher. 2019. The Unfairness of Popularity Bias in Recommendation. arxiv:https:\/\/arXiv.org\/abs\/1907.13286\u00a0[cs.IR] https:\/\/arxiv.org\/abs\/1907.13286"},{"key":"e_1_3_3_2_3_2","series-title":"Proceedings of Machine Learning Research","first-page":"115","volume-title":"Proceedings of the 30th International Conference on Machine Learning","volume":"28","author":"Bergstra James","year":"2013","unstructured":"James Bergstra, Daniel Yamins, and David Cox. 2013. Making a Science of Model Search: Hyperparameter Optimization in Hundreds of Dimensions for Vision Architectures. In Proceedings of the 30th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a028), Sanjoy Dasgupta and David McAllester (Eds.). PMLR, Atlanta, Georgia, USA, 115\u2013123. https:\/\/proceedings.mlr.press\/v28\/bergstra13.html"},{"key":"e_1_3_3_2_4_2","doi-asserted-by":"publisher","DOI":"10.1145\/345508.345543"},{"key":"e_1_3_3_2_5_2","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210014"},{"key":"e_1_3_3_2_6_2","doi-asserted-by":"publisher","DOI":"10.1145\/3341105.3375759"},{"key":"e_1_3_3_2_7_2","doi-asserted-by":"publisher","unstructured":"Diego Carraro and Derek Bridge. 2022. A sampling approach to Debiasing the offline evaluation of recommender systems. J. Intell. Inf. Syst. 58 2 (2022) 311\u2013336. 10.1007\/s10844-021-00651-y","DOI":"10.1007\/s10844-021-00651-y"},{"key":"e_1_3_3_2_8_2","doi-asserted-by":"publisher","unstructured":"Pablo Castells and Alistair Moffat. 2022. Offline Recommender System Evaluation: Challenges and New Directions. AI Magazine 43 2 (Jun. 2022) 225\u2013238. 10.1002\/aaai.12051","DOI":"10.1002\/aaai.12051"},{"key":"e_1_3_3_2_9_2","doi-asserted-by":"publisher","DOI":"10.1145\/3383313.3412259"},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"publisher","unstructured":"Jiawei Chen Hande Dong Xiang Wang Fuli Feng Meng Wang and Xiangnan He. 2023. Bias and Debias in Recommender System: A Survey and Future Directions. ACM Trans. Inf. Syst. 41 3 Article 67 (Feb. 2023) 39\u00a0pages. 10.1145\/3564284","DOI":"10.1145\/3564284"},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"publisher","DOI":"10.1145\/1864708.1864721"},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"publisher","DOI":"10.1145\/3460231.3475943"},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"publisher","DOI":"10.1145\/3383313.3411552"},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"crossref","unstructured":"Miroslav Dud\u00edk Dumitru Erhan John Langford and Lihong Li. 2014. Doubly Robust Policy Evaluation and Optimization. Statist. Sci. 29 4 (2014) 485\u2013511. http:\/\/www.jstor.org\/stable\/43288496","DOI":"10.1214\/14-STS500"},{"key":"e_1_3_3_2_15_2","doi-asserted-by":"publisher","unstructured":"Simen Eide David\u00a0S. Leslie and Arnoldo Frigessi. 2022. Dynamic slate recommendation with gated recurrent units and Thompson sampling. Data Min. Knowl. Discov. 36 5 (sep 2022) 1756\u20131786. 10.1007\/s10618-022-00849-w","DOI":"10.1007\/s10618-022-00849-w"},{"key":"e_1_3_3_2_16_2","doi-asserted-by":"publisher","DOI":"10.1145\/3511808.3557220"},{"key":"e_1_3_3_2_17_2","doi-asserted-by":"publisher","DOI":"10.1145\/3511808.3557624"},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","DOI":"10.1145\/3159652.3159687"},{"key":"e_1_3_3_2_19_2","series-title":"Proceedings of Machine Learning Research","first-page":"3953","volume-title":"Proceedings of the 38th International Conference on Machine Learning","volume":"139","author":"Gupta Shantanu","year":"2021","unstructured":"Shantanu Gupta, Hao Wang, Zachary Lipton, and Yuyang Wang. 2021. Correcting Exposure Bias for Link Recommendation. In Proceedings of the 38th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0139), Marina Meila and Tong Zhang (Eds.). PMLR, 3953\u20133963. https:\/\/proceedings.mlr.press\/v139\/gupta21c.html"},{"key":"e_1_3_3_2_20_2","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531977"},{"key":"e_1_3_3_2_21_2","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657749"},{"key":"e_1_3_3_2_22_2","doi-asserted-by":"publisher","DOI":"10.1109\/WI-IAT59888.2023.00018"},{"key":"e_1_3_3_2_23_2","doi-asserted-by":"publisher","DOI":"10.1145\/3298689.3347069"},{"key":"e_1_3_3_2_24_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4899-7637-6_9"},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","DOI":"10.1145\/1401890.1401944"},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3403226"},{"key":"e_1_3_3_2_27_2","series-title":"CEUR Workshop Proceedings","first-page":"14","volume-title":"Proceedings of the 2nd Workshop on New Trends on Content-Based Recommender Systems co-located with 9th ACM Conference on Recommender Systems (RecSys 2015), Vienna, Austria, September 16-20, 2015.","volume":"1448","author":"Kula Maciej","year":"2015","unstructured":"Maciej Kula. 2015. Metadata Embeddings for User and Item Cold-start Recommendations. In Proceedings of the 2nd Workshop on New Trends on Content-Based Recommender Systems co-located with 9th ACM Conference on Recommender Systems (RecSys 2015), Vienna, Austria, September 16-20, 2015.(CEUR Workshop Proceedings, Vol.\u00a01448), Toine Bogers and Marijn Koolen (Eds.). CEUR-WS.org, 14\u201321. http:\/\/ceur-ws.org\/Vol-1448\/paper4.pdf"},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3403262"},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"publisher","unstructured":"Dong Li Ruoming Jin Zhenming Liu Bin Ren Jing Gao and Zhi Liu. 2024. On Item-Sampling Evaluation for Recommender System. ACM Trans. Recomm. Syst. 2 1 Article 7 (March 2024) 36\u00a0pages. 10.1145\/3629171","DOI":"10.1145\/3629171"},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"publisher","DOI":"10.1145\/3637528.3671915"},{"key":"e_1_3_3_2_31_2","first-page":"108","volume-title":"Causation: Foundation to Application, Workshop at UAI. AUAI","author":"Liang Dawen","year":"2016","unstructured":"Dawen Liang, Laurent Charlin, and David\u00a0M Blei. 2016. Causal inference for recommendation. In Causation: Foundation to Application, Workshop at UAI. AUAI , Vol.\u00a06. 108."},{"key":"e_1_3_3_2_32_2","doi-asserted-by":"publisher","DOI":"10.1145\/2872427.2883090"},{"key":"e_1_3_3_2_33_2","doi-asserted-by":"publisher","DOI":"10.1145\/3485447.3512078"},{"key":"e_1_3_3_2_34_2","doi-asserted-by":"publisher","DOI":"10.1145\/3604915.3610651"},{"key":"e_1_3_3_2_35_2","doi-asserted-by":"publisher","DOI":"10.1145\/3604915.3608809"},{"key":"e_1_3_3_2_36_2","doi-asserted-by":"publisher","DOI":"10.5555\/3020488.3020521"},{"key":"e_1_3_3_2_37_2","doi-asserted-by":"publisher","unstructured":"M.\u00a0E.\u00a0J. Newman. 2005. Power laws Pareto distributions and Zipf\u2019s law. Contemporary Physics 46 5 (2005) 323\u2013351. 10.1080\/00107510500052444 arXiv:10.1080\/00107510500052444","DOI":"10.1080\/00107510500052444"},{"key":"e_1_3_3_2_38_2","doi-asserted-by":"publisher","DOI":"10.1145\/3523227.3551483"},{"key":"e_1_3_3_2_39_2","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3412774"},{"key":"e_1_3_3_2_40_2","doi-asserted-by":"publisher","DOI":"10.1145\/2365952.2365982"},{"key":"e_1_3_3_2_41_2","unstructured":"Steffen Rendle Christoph Freudenthaler Zeno Gantner and Lars Schmidt-Thieme. 2012. BPR: Bayesian Personalized Ranking from Implicit Feedback. arxiv:https:\/\/arXiv.org\/abs\/1205.2618\u00a0[cs.IR] https:\/\/arxiv.org\/abs\/1205.2618"},{"key":"e_1_3_3_2_42_2","doi-asserted-by":"crossref","unstructured":"Paul\u00a0R. Rosenbaum and Donald\u00a0B. Rubin. 1984. Reducing Bias in Observational Studies Using Subclassification on the Propensity Score. J. Amer. Statist. Assoc. 79 387 (1984) 516\u2013524. http:\/\/www.jstor.org\/stable\/2288398","DOI":"10.1080\/01621459.1984.10478078"},{"key":"e_1_3_3_2_43_2","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371783"},{"key":"e_1_3_3_2_44_2","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835542"},{"key":"e_1_3_3_2_45_2","doi-asserted-by":"publisher","DOI":"10.1145\/371920.372071"},{"key":"e_1_3_3_2_46_2","series-title":"Proceedings of Machine Learning Research","first-page":"1670","volume-title":"Proceedings of The 33rd International Conference on Machine Learning","volume":"48","author":"Schnabel Tobias","year":"2016","unstructured":"Tobias Schnabel, Adith Swaminathan, Ashudeep Singh, Navin Chandak, and Thorsten Joachims. 2016. Recommendations as Treatments: Debiasing Learning and Evaluation. In Proceedings of The 33rd International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a048), Maria\u00a0Florina Balcan and Kilian\u00a0Q. Weinberger (Eds.). PMLR, New York, New York, USA, 1670\u20131679. https:\/\/proceedings.mlr.press\/v48\/schnabel16.html"},{"key":"e_1_3_3_2_47_2","doi-asserted-by":"publisher","unstructured":"Monika Singh. 2020. Scalability and sparsity issues in recommender datasets: a survey. Knowl. Inf. Syst. 62 1 (Jan. 2020) 1\u201343. 10.1007\/s10115-018-1254-2","DOI":"10.1007\/s10115-018-1254-2"},{"key":"e_1_3_3_2_48_2","doi-asserted-by":"publisher","DOI":"10.1145\/1835804.1835895"},{"key":"e_1_3_3_2_49_2","doi-asserted-by":"publisher","DOI":"10.1145\/2507157.2507160"},{"key":"e_1_3_3_2_50_2","doi-asserted-by":"publisher","DOI":"10.1145\/2365952.2365972"},{"key":"e_1_3_3_2_51_2","doi-asserted-by":"publisher","DOI":"10.1145\/3240323.3240347"},{"key":"e_1_3_3_2_52_2","unstructured":"Chengbing Wang Wentao Shi Jizhi Zhang Wenjie Wang Hang Pan and Fuli Feng. 2024. Debias Can be Unreliable: Mitigating Bias Issue in Evaluating Debiasing Recommendation. arxiv:https:\/\/arXiv.org\/abs\/2409.04810\u00a0[cs.IR] https:\/\/arxiv.org\/abs\/2409.04810"},{"key":"e_1_3_3_2_53_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.331"},{"key":"e_1_3_3_2_54_2","doi-asserted-by":"publisher","unstructured":"Eva Zangerle and Christine Bauer. 2022. Evaluating Recommender Systems: Survey and Framework. ACM Comput. Surv. 55 8 Article 170 (Dec. 2022) 38\u00a0pages. 10.1145\/3556536","DOI":"10.1145\/3556536"}],"event":{"name":"RecSys '25: Nineteenth ACM Conference on Recommender Systems","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction","SIGAI ACM Special Interest Group on Artificial Intelligence","SIGIR ACM Special Interest Group on Information Retrieval","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data","SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"],"location":"Prague Czech Republic","acronym":"RecSys '25"},"container-title":["Proceedings of the Nineteenth ACM Conference on Recommender Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3705328.3748086","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,6]],"date-time":"2025-09-06T11:48:27Z","timestamp":1757159307000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3705328.3748086"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,9,7]]},"references-count":53,"alternative-id":["10.1145\/3705328.3748086","10.1145\/3705328"],"URL":"https:\/\/doi.org\/10.1145\/3705328.3748086","relation":{},"subject":[],"published":{"date-parts":[[2025,9,7]]},"assertion":[{"value":"2025-09-07","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}