{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,3]],"date-time":"2025-12-03T18:06:48Z","timestamp":1764785208520,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":87,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,9,14]],"date-time":"2023-09-14T00:00:00Z","timestamp":1694649600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,9,14]]},"DOI":"10.1145\/3604915.3608809","type":"proceedings-article","created":{"date-parts":[[2023,9,14]],"date-time":"2023-09-14T22:40:23Z","timestamp":1694731223000},"page":"658-670","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["What We Evaluate When We Evaluate Recommender Systems: Understanding Recommender Systems\u2019 Performance using Item Response Theory"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1454-9585","authenticated-orcid":false,"given":"Yang","family":"Liu","sequence":"first","affiliation":[{"name":"Department of Computer Science, University of Helsinki, Finland"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5139-9483","authenticated-orcid":false,"given":"Alan","family":"Medlar","sequence":"additional","affiliation":[{"name":"Department of Computer Science, University of Helsinki, Finland"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7008-3380","authenticated-orcid":false,"given":"Dorota","family":"Glowacka","sequence":"additional","affiliation":[{"name":"Department of Computer Science, University of Helsinki, Finland"}]}],"member":"320","published-online":{"date-parts":[[2023,9,14]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.5555\/2931100"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.3390\/a11090137"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3503252.3531292"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132847.3133083"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-017-9312-z"},{"key":"e_1_3_2_1_6_1","volume-title":"Graph convolutional matrix completion. arXiv preprint arXiv:1706.02263","author":"van\u00a0den Berg Rianne","year":"2017","unstructured":"Rianne van\u00a0den Berg, Thomas\u00a0N Kipf, and Max Welling. 2017. Graph convolutional matrix completion. arXiv preprint arXiv:1706.02263 (2017)."},{"key":"e_1_3_2_1_7_1","volume-title":"Some latent trait models and their use in inferring an examinee\u2019s ability. Statistical Theories of Mental Test Scores","author":"Birnbaum Allan","year":"1968","unstructured":"Allan Birnbaum. 1968. Some latent trait models and their use in inferring an examinee\u2019s ability. Statistical Theories of Mental Test Scores (1968)."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.18637\/jss.v100.i05"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3383313.3412259"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313705"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3373807","article-title":"Efficient neural matrix factorization without sampling for recommendation","volume":"38","author":"Chen Chong","year":"2020","unstructured":"Chong Chen, Min Zhang, Yongfeng Zhang, Yiqun Liu, and Shaoping Ma. 2020. Efficient neural matrix factorization without sampling for recommendation. ACM Transactions on Information Systems 38, 2 (2020), 1\u201328.","journal-title":"ACM Transactions on Information Systems"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2988450.2988454"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1864708.1864721"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3460231.3475943"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/963770.963776"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3434185"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1176\/ps.2008.59.4.361"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2017\/239"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2872427.2883037"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401063"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2018\/308"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2018.2831682"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3038912.3052569"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/963770.963772"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2505515.2505665"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1007665907178"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2959100.2959134"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2487575.2487589"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2926720"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3576840.3578310"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3403226"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1500"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1062"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1434"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1434"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3219819.3220023"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3178876.3186150"},{"key":"e_1_3_2_1_39_1","volume-title":"David\u00a0B Pillemer","author":"Light J","year":"1984","unstructured":"Richard\u00a0J Light, J Richard, Richard Light, David\u00a0B Pillemer, 1984. Summing up: The science of reviewing research. Harvard University Press."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3485447.3512104"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3604915.3610651"},{"key":"e_1_3_2_1_42_1","volume-title":"Towards amortized ranking-critical training for collaborative filtering. arXiv preprint arXiv:1906.04281","author":"Lobel Sam","year":"2019","unstructured":"Sam Lobel, Chunyuan Li, Jianfeng Gao, and Lawrence Carin. 2019. Towards amortized ranking-critical training for collaborative filtering. arXiv preprint arXiv:1906.04281 (2019)."},{"key":"e_1_3_2_1_43_1","volume-title":"Learning disentangled representations for recommendation. Advances in Neural Information Processing Systems 32","author":"Ma Jianxin","year":"2019","unstructured":"Jianxin Ma, Chang Zhou, Peng Cui, Hongxia Yang, and Wenwu Zhu. 2019. Learning disentangled representations for recommendation. Advances in Neural Information Processing Systems 32 (2019)."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3459637.3482297"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2018.09.004"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"crossref","unstructured":"Sean\u00a0M McNee John Riedl and Joseph\u00a0A Konstan. 2006. Being accurate is not enough: how accuracy metrics have hurt recommender systems. In CHI Extended Abstracts on Human Factors in Computing Systems. 1097\u20131101.","DOI":"10.1145\/1125451.1125659"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2021.108076"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2011.134"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3178876.3186040"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2016.0151"},{"volume-title":"Probabilistic models for some intelligence and attainment tests","author":"Rasch Georg","key":"e_1_3_2_1_51_1","unstructured":"Georg Rasch. 1960. Probabilistic models for some intelligence and attainment tests. Danish Institute for Educational Research."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2010.127"},{"key":"e_1_3_2_1_53_1","volume-title":"Proceedings of the 25th Conference on Uncertainty in Artificial Intelligence. 452\u2013461","author":"Rendle Steffen","year":"2009","unstructured":"Steffen Rendle, Christoph Freudenthaler, Zeno Gantner, and Lars Schmidt-Thieme. 2009. BPR: Bayesian personalized ranking from implicit feedback. In Proceedings of the 25th Conference on Uncertainty in Artificial Intelligence. 452\u2013461."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/1242572.1242643"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.346"},{"volume-title":"Modern psychometrics: The science of psychological assessment","author":"Rust John","key":"e_1_3_2_1_56_1","unstructured":"John Rust and Susan Golombok. 2014. Modern psychometrics: The science of psychological assessment. Routledge."},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371831"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1007\/s13042-017-0762-9"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/3357384.3357925"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313710"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371774"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3383313.3412489"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/2736277.2741093"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3240323.3240347"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330836"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313411"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313417"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3124749.3124754"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3450078"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330989"},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331267"},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3450133"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401137"},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.1177\/014662168200600408"},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.1080\/07481756.2004.11909751"},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331201"},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462862"},{"key":"e_1_3_2_1_78_1","volume-title":"Variational item response theory: Fast, accurate, and expressive. arXiv preprint arXiv:2002.00276","author":"Wu Mike","year":"2020","unstructured":"Mike Wu, Richard\u00a0L Davis, Benjamin\u00a0W Domingue, Chris Piech, and Noah Goodman. 2020. Variational item response theory: Fast, accurate, and expressive. arXiv preprint arXiv:2002.00276 (2020)."},{"key":"e_1_3_2_1_79_1","doi-asserted-by":"publisher","DOI":"10.1145\/2835776.2835837"},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2017\/435"},{"key":"e_1_3_2_1_81_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2017\/447"},{"key":"e_1_3_2_1_82_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939673"},{"key":"e_1_3_2_1_83_1","volume-title":"Proceedings of the 38th European Conference on Information Retrieval Research. 45\u201357","author":"Zhang Weinan","year":"2016","unstructured":"Weinan Zhang, Tianming Du, and Jun Wang. 2016. Deep Learning over Multi-field Categorical Data: \u2013A Case Study on User Response Prediction. In Proceedings of the 38th European Conference on Information Retrieval Research. 45\u201357."},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3545796","article-title":"A revisiting study of appropriate offline evaluation for top-N recommendation algorithms","volume":"41","author":"Zhao Wayne\u00a0Xin","year":"2022","unstructured":"Wayne\u00a0Xin Zhao, Zihan Lin, Zhichao Feng, Pengfei Wang, and Ji-Rong Wen. 2022. A revisiting study of appropriate offline evaluation for top-N recommendation algorithms. ACM Transactions on Information Systems 41, 2 (2022), 1\u201341.","journal-title":"ACM Transactions on Information Systems"},{"key":"e_1_3_2_1_85_1","doi-asserted-by":"publisher","DOI":"10.1145\/3459637.3482016"},{"key":"e_1_3_2_1_86_1","doi-asserted-by":"publisher","DOI":"10.1145\/3240323.3240343"},{"key":"e_1_3_2_1_87_1","doi-asserted-by":"publisher","DOI":"10.1177\/1533317514522852"}],"event":{"name":"RecSys '23: Seventeenth ACM Conference on Recommender Systems","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGAI ACM Special Interest Group on Artificial Intelligence","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data","SIGIR ACM Special Interest Group on Information Retrieval","SIGCHI ACM Special Interest Group on Computer-Human Interaction","SIGecom Special Interest Group on Economics and Computation"],"location":"Singapore Singapore","acronym":"RecSys '23"},"container-title":["Proceedings of the 17th ACM Conference on Recommender Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3604915.3608809","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3604915.3608809","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:46:06Z","timestamp":1750178766000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3604915.3608809"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,9,14]]},"references-count":87,"alternative-id":["10.1145\/3604915.3608809","10.1145\/3604915"],"URL":"https:\/\/doi.org\/10.1145\/3604915.3608809","relation":{},"subject":[],"published":{"date-parts":[[2023,9,14]]},"assertion":[{"value":"2023-09-14","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}