{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,13]],"date-time":"2026-10-13T17:31:39Z","timestamp":1791912699697,"version":"4.3.4"},"reference-count":74,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2020,6,8]],"date-time":"2020-06-08T00:00:00Z","timestamp":1591574400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2020,6,8]],"date-time":"2020-06-08T00:00:00Z","timestamp":1591574400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/100014440","name":"Ministerio de Ciencia, Innovaci\u00f3n y Universidades","doi-asserted-by":"publisher","award":["TIN2016-80630-P"],"award-info":[{"award-number":["TIN2016-80630-P"]}],"id":[{"id":"10.13039\/100014440","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100014440","name":"Ministerio de Ciencia, Innovaci\u00f3n y Universidades","doi-asserted-by":"publisher","award":["RTI2018-093336-B-C22"],"award-info":[{"award-number":["RTI2018-093336-B-C22"]}],"id":[{"id":"10.13039\/100014440","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100008425","name":"Conseller\u00eda de Cultura, Educaci\u00f3n e Ordenaci\u00f3n Universitaria, Xunta de Galicia","doi-asserted-by":"publisher","award":["ED431G\/01"],"award-info":[{"award-number":["ED431G\/01"]}],"id":[{"id":"10.13039\/501100008425","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100008425","name":"Conseller\u00eda de Cultura, Educaci\u00f3n e Ordenaci\u00f3n Universitaria, Xunta de Galicia","doi-asserted-by":"publisher","award":["ED431B 2019\/03"],"award-info":[{"award-number":["ED431B 2019\/03"]}],"id":[{"id":"10.13039\/501100008425","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval J"],"published-print":{"date-parts":[[2020,8]]},"DOI":"10.1007\/s10791-020-09377-x","type":"journal-article","created":{"date-parts":[[2020,6,8]],"date-time":"2020-06-08T07:05:49Z","timestamp":1591599949000},"page":"411-448","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":74,"title":["Assessing ranking metrics in top-N recommendation"],"prefix":"10.1007","volume":"23","author":[{"given":"Daniel","family":"Valcarce","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alejandro","family":"Bellog\u00edn","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5997-8252","authenticated-orcid":false,"given":"Javier","family":"Parapar","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Pablo","family":"Castells","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2020,6,8]]},"reference":[{"issue":"1","key":"9377_CR1","doi-asserted-by":"publisher","first-page":"2","DOI":"10.1145\/2215676.2215678","volume":"46","author":"J Allan","year":"2012","unstructured":"Allan, J., Croft, B., Moffat, A., & Sanderson, M. (2012). Frontiers, challenges, and opportunities for information retrieval: Report from SWIRL 2012 the second strategic workshop on information retrieval in lorne. SIGIR Forum, 46(1), 2\u201332. https:\/\/doi.org\/10.1145\/2215676.2215678.","journal-title":"SIGIR Forum"},{"key":"9377_CR2","volume-title":"The Long Tail: Why the Future of Business Is Selling Less of More","author":"C Anderson","year":"2008","unstructured":"Anderson, C. (2008). The Long Tail: Why the Future of Business Is Selling Less of More. New York: Hachette Books."},{"key":"9377_CR3","doi-asserted-by":"publisher","unstructured":"Beel, J., & Langer, S. (2015). A comparison of offline evaluations, online evaluations, and user studies in the context of research-paper recommender systems. In S. Kapidakis, C. Mazurek, M. Werla (eds.) Proceedings of the 19th international conference on theory and practice of digital libraries, TPDL \u201915 (pp. 153\u2013168). Springer, Cham. https:\/\/doi.org\/10.1007\/978-3-319-24592-8_12.","DOI":"10.1007\/978-3-319-24592-8_12"},{"issue":"12","key":"9377_CR4","doi-asserted-by":"publisher","first-page":"29","DOI":"10.1145\/138859.138861","volume":"35","author":"NJ Belkin","year":"1992","unstructured":"Belkin, N. J., & Croft, W. B. (1992). Information filtering and information retrieval: Two sides of the same coin? Communications of the ACM, 35(12), 29\u201338. https:\/\/doi.org\/10.1145\/138859.138861.","journal-title":"Communications of the ACM"},{"key":"9377_CR5","doi-asserted-by":"publisher","unstructured":"Bellog\u00edn, A., Castells, P., & Cantador, I. (2011). Precision-oriented evaluation of recommender systems. In Proceedings of the 5th ACM conference on recommender systems, RecSys \u201911 (p. 333). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/2043932.2043996.","DOI":"10.1145\/2043932.2043996"},{"issue":"6","key":"9377_CR6","doi-asserted-by":"publisher","first-page":"606","DOI":"10.1007\/s10791-017-9312-z","volume":"20","author":"A Bellog\u00edn","year":"2017","unstructured":"Bellog\u00edn, A., Castells, P., & Cantador, I. (2017). Statistical biases in information retrieval metrics for recommender systems. Information Retrieval Journal, 20(6), 606\u2013634. https:\/\/doi.org\/10.1007\/s10791-017-9312-z.","journal-title":"Information Retrieval Journal"},{"issue":"6","key":"9377_CR7","doi-asserted-by":"publisher","first-page":"697","DOI":"10.1007\/s10791-012-9214-z","volume":"16","author":"A Bellog\u00edn","year":"2013","unstructured":"Bellog\u00edn, A., Wang, J., & Castells, P. (2013). Bridging memory-based collaborative filtering and text retrieval. Information Retrieval, 16(6), 697\u2013724. https:\/\/doi.org\/10.1007\/s10791-012-9214-z.","journal-title":"Information Retrieval"},{"key":"9377_CR8","doi-asserted-by":"crossref","unstructured":"Bennett, J., & Lanning, S (2007). The netflix prize. In Proceedings of the KDD cup workshop 2007 (pp. 3\u20136). ACM, New York, NY, USA.","DOI":"10.1145\/1345448.1345459"},{"key":"9377_CR9","first-page":"993","volume":"3","author":"DM Blei","year":"2003","unstructured":"Blei, D. M., Ng, A. Y., & Jordan, M. I. (2003). Latent Dirichlet allocation. Journal of Machine Learning Research, 3, 993\u20131022.","journal-title":"Journal of Machine Learning Research"},{"issue":"6","key":"9377_CR10","doi-asserted-by":"publisher","first-page":"491","DOI":"10.1007\/s10791-007-9032-x","volume":"10","author":"C Buckley","year":"2007","unstructured":"Buckley, C., Dimmick, D., Soboroff, I., & Voorhees, E. (2007). Bias and the limits of pooling for large collections. Information Retrieval, 10(6), 491\u2013508. https:\/\/doi.org\/10.1007\/s10791-007-9032-x.","journal-title":"Information Retrieval"},{"key":"9377_CR11","doi-asserted-by":"publisher","unstructured":"Buckley, C., & Voorhees, E.M (2000). Evaluating evaluation measure stability. In Proceedings of the 23rd annual international ACM SIGIR conference on research and development in information retrieval, SIGIR \u201900 (pp. 33\u201340). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/345508.345543.","DOI":"10.1145\/345508.345543"},{"key":"9377_CR12","doi-asserted-by":"publisher","unstructured":"Buckley, C., & Voorhees, E.M. (2004). Retrieval evaluation with incomplete information. In Proceedings of the 27th annual international ACM SIGIR conference on research and development in information retrieval, SIGIR \u201904 (pp. 25\u201332). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/1008992.1009000.","DOI":"10.1145\/1008992.1009000"},{"key":"9377_CR13","doi-asserted-by":"publisher","unstructured":"B\u00fcttcher, S., Clarke, C.L.A., Yeung, P.C.K., & Soboroff, I. (2007). Reliable information retrieval evaluation with incomplete and biased judgements. In Proceedings of the 30th annual international ACM SIGIR conference on research and development in information retrieval, SIGIR \u201907, (pp. 63\u201370). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/1277741.1277755.","DOI":"10.1145\/1277741.1277755"},{"issue":"1\u20132","key":"9377_CR14","doi-asserted-by":"publisher","first-page":"67","DOI":"10.1007\/s11257-012-9136-x","volume":"24","author":"PG Campos","year":"2014","unstructured":"Campos, P. G., D\u00edez, F., & Cantador, I. (2014). Time-aware recommender systems: a comprehensive survey and analysis of existing evaluation protocols. User Model. User-Adapt. Interact., 24(1\u20132), 67\u2013119. https:\/\/doi.org\/10.1007\/s11257-012-9136-x.","journal-title":"User Model. User-Adapt. Interact."},{"key":"9377_CR15","doi-asserted-by":"publisher","unstructured":"Ca\u00f1amares, R., & Castells, P. (2018). Should i follow the crowd? A probabilistic analysis of the effectiveness of popularity in recommender systems. In Proceedings of the 41st International ACM SIGIR Conference on Research & Development in Information Retrieval, SIGIR \u201918 (pp. 415\u2013424). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/3209978.3210014.","DOI":"10.1145\/3209978.3210014"},{"key":"9377_CR16","doi-asserted-by":"publisher","unstructured":"Castells, P., Hurley, N.J., & Vargas, S. (2015). Novelty and diversity in recommender systems. In F. Ricci, L. Rokach, B. Shapira (eds.) Recommender Systems Handbook, 2nd edn. (pp. 881\u2013918). Springer, Boston, MA. https:\/\/doi.org\/10.1007\/978-1-4899-7637-6_26.","DOI":"10.1007\/978-1-4899-7637-6_26"},{"key":"9377_CR17","doi-asserted-by":"publisher","unstructured":"Chapelle, O., Metlzer, D., Zhang, Y., & Grinspan, P. (2009). Expected reciprocal rank for graded relevance. In Proceedings of the 18th ACM conference on information and knowledge management, CIKM \u201909 (p. 621). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/1645953.1646033.","DOI":"10.1145\/1645953.1646033"},{"key":"9377_CR18","doi-asserted-by":"publisher","unstructured":"Cremonesi, P., Garzotto, F., Negro, S., Papadopoulos, A.V., & Turrin, R. (2011). Looking for \u201cGood\u201d recommendations: A comparative evaluation of recommender systems. In Proceedings of the 13th IFIP TC 13 international conference on human-computer interaction\u2013Part III, INTERACT \u201911 (pp. 152\u2013168). Springer, Berlin. https:\/\/doi.org\/10.1007\/978-3-642-23765-2_11.","DOI":"10.1007\/978-3-642-23765-2_11"},{"key":"9377_CR19","doi-asserted-by":"publisher","unstructured":"Cremonesi, P., Garzotto, F., & Turrin, R. (2013). User-centric vs. system-centric evaluation of recommender systems. In Proceedings of the 14th IFIP TC 13 international conference on human-computer interaction\u2013Part III, INTERACT \u201913 (pp. 334\u2013351). Springer, Berlin. https:\/\/doi.org\/10.1007\/978-3-642-40477-1_21.","DOI":"10.1007\/978-3-642-40477-1_21"},{"key":"9377_CR20","doi-asserted-by":"publisher","unstructured":"Cremonesi, P., Koren, Y., & Turrin, R. (2010). Performance of recommender algorithms on top-N recommendation tasks. In Proceedings of the 4th ACM conference on recommender systems, RecSys \u201910 (pp. 39\u201346). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/1864708.1864721.","DOI":"10.1145\/1864708.1864721"},{"key":"9377_CR21","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4899-4541-9","volume-title":"An introduction to the bootstrap, monographs on statistics and applied probability","author":"B Efron","year":"1993","unstructured":"Efron, B., & Tibshirani, R. J. (1993). An introduction to the bootstrap, monographs on statistics and applied probability. Boca Raton, Florida, USA: Chapman & Hall\/CRC."},{"issue":"1","key":"9377_CR22","doi-asserted-by":"publisher","first-page":"91","DOI":"10.1145\/3274784.3274789","volume":"52","author":"N Ferro","year":"2018","unstructured":"Ferro, N., Fuhr, N., Grefenstette, G., Konstan, J. A., Castells, P., Daly, E. M., et al. (2018). The dagstuhl perspectives workshop on performance modeling and prediction. SIGIR Forum, 52(1), 91\u2013101. https:\/\/doi.org\/10.1145\/3274784.3274789.","journal-title":"SIGIR Forum"},{"issue":"3","key":"9377_CR23","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1145\/3190580.3190586","volume":"51","author":"N Fuhr","year":"2018","unstructured":"Fuhr, N. (2018). Some common mistakes in IR evaluation, and how they can be avoided. SIGIR Forum, 51(3), 32\u201341. https:\/\/doi.org\/10.1145\/3190580.3190586.","journal-title":"SIGIR Forum"},{"key":"9377_CR24","doi-asserted-by":"publisher","unstructured":"Garcin, F., Faltings, B., Donatsch, O., Alazzawi, A., Bruttin, C., & Huber, A. (2014). Offline and online evaluation of news recommender systems at swissinfo.ch. In Proceedings of the 8th ACM conference on recommender systems, RecSys \u201914 (pp. 169\u2013176). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/2645710.2645745.","DOI":"10.1145\/2645710.2645745"},{"key":"9377_CR25","doi-asserted-by":"crossref","unstructured":"Gilotte, A., Calauz\u00e8nes, C., Nedelec, T., Abraham, A., & Doll\u00e9, S. (2018). Offline A\/B testing for recommender systems. In Proceedings of the 11th ACM international conference on web search and data mining, WSDM \u201918 (pp. 198\u2013206). ACM.","DOI":"10.1145\/3159652.3159687"},{"key":"9377_CR26","volume-title":"Variabilit\u00e0 e Mutuabilit\u00e0","author":"C Gini","year":"1912","unstructured":"Gini, C. (1912). Variabilit\u00e0 e Mutuabilit\u00e0. Cuppini: Contributo allo Studio delle Distribuzioni e delle Relazioni Statistiche."},{"key":"9377_CR27","doi-asserted-by":"crossref","unstructured":"Gruson, A., Chandar, P., Charbuillet, C., McInerney, J., Hansen, S., Tardieu, D., & Carterette, B. (2019). Offline evaluation to make decisions about playlist recommendation. In Proceedings of the 12th ACM international conference on web search and data mining, WSDM \u201919 (pp. 420\u2013428). ACM.","DOI":"10.1145\/3289600.3291027"},{"key":"9377_CR28","doi-asserted-by":"publisher","unstructured":"Gunawardana, A., & Shani, G. (2015). Evaluating recommender systems. In: F. Ricci, L. Rokach, B. Shapira (eds.) Recommender systems handbook, 2nd edn. (pp. 265\u2013308). Springer, Boston, MA, USA. https:\/\/doi.org\/10.1007\/978-1-4899-7637-6_8.","DOI":"10.1007\/978-1-4899-7637-6_8"},{"issue":"4","key":"9377_CR29","doi-asserted-by":"publisher","first-page":"19:1","DOI":"10.1145\/2827872","volume":"5","author":"FM Harper","year":"2015","unstructured":"Harper, F. M., & Konstan, J. A. (2015). The movielens datasets: History and context. Acm Transactions on Interactive Intelligent Systems, 5(4), 19:1\u201319:19. https:\/\/doi.org\/10.1145\/2827872.","journal-title":"Acm Transactions on Interactive Intelligent Systems"},{"issue":"1","key":"9377_CR30","doi-asserted-by":"publisher","first-page":"5","DOI":"10.1145\/963770.963772","volume":"22","author":"JL Herlocker","year":"2004","unstructured":"Herlocker, J. L., Konstan, J. A., Terveen, L. G., & Riedl, J. T. (2004). Evaluating collaborative filtering recommender systems. ACM Transactions on Information Systems, 22(1), 5\u201353. https:\/\/doi.org\/10.1145\/963770.963772.","journal-title":"ACM Transactions on Information Systems"},{"issue":"1","key":"9377_CR31","doi-asserted-by":"publisher","first-page":"89","DOI":"10.1145\/963770.963774","volume":"22","author":"T Hofmann","year":"2004","unstructured":"Hofmann, T. (2004). Latent semantic models for collaborative filtering. ACM Transactions on Information Systems, 22(1), 89\u2013115. https:\/\/doi.org\/10.1145\/963770.963774.","journal-title":"ACM Transactions on Information Systems"},{"issue":"4","key":"9377_CR32","doi-asserted-by":"publisher","first-page":"805","DOI":"10.2139\/ssrn.1321962","volume":"60","author":"K Hosanagar","year":"2014","unstructured":"Hosanagar, K., Fleder, D., Lee, D., & Buja, A. (2014). Will the global village fracture into tribes: Recommender systems and their effects on consumers. Management Science, 60(4), 805\u2013823. https:\/\/doi.org\/10.2139\/ssrn.1321962.","journal-title":"Management Science"},{"key":"9377_CR33","doi-asserted-by":"publisher","unstructured":"Hu, Y., Koren, Y., & Volinsky, C. (2008). Collaborative filtering for implicit feedback datasets. In Proceedings of the 2008 eighth IEEE international conference on data mining, ICDM \u201908 (pp. 263\u2013272). IEEE, Washington, DC, USA. https:\/\/doi.org\/10.1109\/ICDM.2008.22.","DOI":"10.1109\/ICDM.2008.22"},{"issue":"4","key":"9377_CR34","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin, K., & Kek\u00e4l\u00e4inen, J. (2002). Cumulated gain-based evaluation of IR techniques. ACM Transactions on Information Systems, 20(4), 422\u2013446. https:\/\/doi.org\/10.1145\/582415.582418.","journal-title":"ACM Transactions on Information Systems"},{"key":"9377_CR35","doi-asserted-by":"publisher","unstructured":"Joachims, T., Swaminathan, A., & Schnabel, T. (2017). Unbiased learning-to-rank with biased feedback. In Proceedings of the 10th ACM International Conference on Web Search and Data Mining, WSDM \u201917 (pp. 781\u2013789). ACM. https:\/\/doi.org\/10.1145\/3018661.3018699.","DOI":"10.1145\/3018661.3018699"},{"issue":"1\u20132","key":"9377_CR36","doi-asserted-by":"publisher","first-page":"81","DOI":"10.1093\/biomet\/30.1-2.81","volume":"30","author":"MG Kendall","year":"1938","unstructured":"Kendall, M. G. (1938). A new measure of rank correlation. Biometrika, 30(1\u20132), 81\u201393. https:\/\/doi.org\/10.1093\/biomet\/30.1-2.81.","journal-title":"Biometrika"},{"issue":"1","key":"9377_CR37","doi-asserted-by":"publisher","first-page":"140","DOI":"10.1007\/s10618-008-0114-1","volume":"18","author":"R Kohavi","year":"2009","unstructured":"Kohavi, R., Longbotham, R., Sommerfield, D., & Henne, R. M. (2009). Controlled experiments on the web: Survey and practical guide. Data Mining and Knowledge Discovery, 18(1), 140\u2013181. https:\/\/doi.org\/10.1007\/s10618-008-0114-1.","journal-title":"Data Mining and Knowledge Discovery"},{"key":"9377_CR38","doi-asserted-by":"publisher","unstructured":"Konstan, J.A., & Adomavicius, G. (2013). Toward identification and adoption of best practices in algorithmic recommender systems research. In Proceedings of the international workshop on reproducibility and replication in recommender systems evaluation, RepSys \u201913 (pp. 23\u201328). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/2532508.2532513.","DOI":"10.1145\/2532508.2532513"},{"issue":"1","key":"9377_CR39","doi-asserted-by":"publisher","first-page":"37","DOI":"10.1016\/j.ipm.2017.09.002","volume":"54","author":"M Kutlu","year":"2018","unstructured":"Kutlu, M., Elsayed, T., & Lease, M. (2018). Intelligent topic selection for low-cost information retrieval evaluation: A New perspective on deep vs. shallow judging. Information Processing & Management, 54(1), 37\u201359. https:\/\/doi.org\/10.1016\/j.ipm.2017.09.002.","journal-title":"Information Processing & Management"},{"issue":"5","key":"9377_CR40","doi-asserted-by":"publisher","first-page":"1005","DOI":"10.1016\/j.ipm.2017.04.005","volume":"53","author":"DE Losada","year":"2017","unstructured":"Losada, D. E., Parapar, J., & Barreiro, A. (2017). Multi-armed bandits for adjudicating documents in pooling-based evaluation of information retrieval systems. Information Processing & Management, 53(5), 1005\u20131025. https:\/\/doi.org\/10.1016\/j.ipm.2017.04.005.","journal-title":"Information Processing & Management"},{"issue":"4","key":"9377_CR41","doi-asserted-by":"publisher","first-page":"416","DOI":"10.1007\/s10791-016-9282-6","volume":"19","author":"X Lu","year":"2016","unstructured":"Lu, X., Moffat, A., & Culpepper, J. S. (2016). The effect of pooling and evaluation depth on IR metrics. Information Retrieval Journal, 19(4), 416\u2013445. https:\/\/doi.org\/10.1007\/s10791-016-9282-6.","journal-title":"Information Retrieval Journal"},{"key":"9377_CR42","unstructured":"Marlin, B. M., Zemel, R. S., Roweis, S., & Slaney, M. (2007). Collaborative filtering and the missing at random assumption. In Proceedings of the 23rd conference on uncertainty in artificial intelligence, UAI\u201907 (pp. 267\u2013275). Arlington, VA: AUAI Press."},{"issue":"2","key":"9377_CR43","first-page":"85","volume":"3","author":"O Matos-Junior","year":"2012","unstructured":"Matos-Junior, O., Ziviani, N., Botelho, F., Cristo, M., Lacerda, A., & da Silva, A. S. (2012). Using taxonomies for product recommendation. Journal of Information and Data Management, 3(2), 85\u2013100.","journal-title":"Journal of Information and Data Management"},{"key":"9377_CR44","doi-asserted-by":"publisher","unstructured":"McNee, S. M., Riedl, J. T., & Konstan, J. A. (2006). Being accurate is not enough: How accuracy metrics have hurt recommender systems. In CHI \u201906 extended abstracts on human factors in computing systems, CHI EA \u201906 (p. 1097). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/1125451.1125659.","DOI":"10.1145\/1125451.1125659"},{"key":"9377_CR45","doi-asserted-by":"publisher","unstructured":"Ning, X., & Karypis, G. (2011). SLIM: sparse linear methods for top-N recommender systems. In Proceedings of the 2011 IEEE 11th international conference on data mining, ICDM \u201911 (pp. 497\u2013506). IEEE, Washington, DC, USA. https:\/\/doi.org\/10.1109\/ICDM.2011.134.","DOI":"10.1109\/ICDM.2011.134"},{"issue":"4","key":"9377_CR46","doi-asserted-by":"publisher","first-page":"966","DOI":"10.1016\/j.ipm.2013.03.001","volume":"49","author":"J Parapar","year":"2013","unstructured":"Parapar, J., Bellog\u00edn, A., Castells, P., & Barreiro, \u00c1. (2013). Relevance-based Language modelling for recommender systems. Information Processing & Management, 49(4), 966\u2013980. https:\/\/doi.org\/10.1016\/j.ipm.2013.03.001.","journal-title":"Information Processing & Management"},{"key":"9377_CR47","doi-asserted-by":"publisher","unstructured":"Parapar, J., Losada, D.E., PresedoQuindimil, M.A., & Barreiro, \u00c1. (2019). Using score distributions to compare statistical significance tests for information retrieval evaluation. Journal of the Association for Information Science and Technology. https:\/\/doi.org\/10.1002\/asi.24203","DOI":"10.1002\/asi.24203"},{"key":"9377_CR48","doi-asserted-by":"publisher","unstructured":"Park, S.-T., Chu, W., Park, S.T., & Chu, W. (2009). Pairwise preference regression for cold-start recommendation. In Proceedings of the 3rd ACM conference on recommender systems, RecSys \u201909 (pp. 21\u201328). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/1639714.1639720.","DOI":"10.1145\/1639714.1639720"},{"key":"9377_CR49","unstructured":"Rendle, S., Freudenthaler, C., Gantner, Z., & Schmidt-Thieme, L. (2009). BPR: Bayesian personalized ranking from implicit feedback. In Proceedings of the twenty-fifth conference on uncertainty in artificial intelligence, UAI \u201909 (pp. 452\u2013461). AUAI Press, Arlington, VA, US."},{"key":"9377_CR50","doi-asserted-by":"publisher","unstructured":"Ricci, F., Rokach, L., & Shapira, B. (2015). Recommender systems handbook, 2nd edn. Boston, MA: Springer. https:\/\/doi.org\/10.1007\/978-1-4899-7637-6.","DOI":"10.1007\/978-1-4899-7637-6"},{"key":"9377_CR51","doi-asserted-by":"publisher","unstructured":"Robertson, S. (2006). On GMAP: And other transformations. In Proceedings of the 15th ACM international conference on information and knowledge management, CIKM \u201906 (pp. 78\u201383). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/1183614.1183630.","DOI":"10.1145\/1183614.1183630"},{"key":"9377_CR52","doi-asserted-by":"publisher","unstructured":"Rossetti, M., Stella, F., & Zanker, M. (2016). Contrasting offline and online results when evaluating recommendation algorithms. In Proceedings of the 10th ACM conference on recommender systems, RecSys \u201916 (pp. 31\u201334). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/2959100.2959176.","DOI":"10.1145\/2959100.2959176"},{"key":"9377_CR53","doi-asserted-by":"publisher","unstructured":"Said, A., & Bellog\u00edn, A. (2014). Comparative recommender system evaluation: Benchmarking recommendation frameworks. In Proceedings of the 8th ACM conference on recommender systems, RecSys \u201914 (pp. 129\u2013136). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/2645710.2645746.","DOI":"10.1145\/2645710.2645746"},{"key":"9377_CR54","doi-asserted-by":"publisher","unstructured":"Sakai, T. (2006). Evaluating evaluation metrics based on the bootstrap. In Proceedings of the 29th annual international ACM SIGIR conference on research and development in information retrieval, SIGIR \u201906 (pp. 525\u2013532). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/1148170.1148261.","DOI":"10.1145\/1148170.1148261"},{"key":"9377_CR55","doi-asserted-by":"publisher","unstructured":"Sakai, T. (2012). Evaluation with informational and navigational intents. In Proceedings of the 21st international conference on world wide web, WWW \u201912 (pp. 499\u2013508). New York, NY: ACM. https:\/\/doi.org\/10.1145\/2187836.2187904.","DOI":"10.1145\/2187836.2187904"},{"issue":"5","key":"9377_CR56","doi-asserted-by":"publisher","first-page":"447","DOI":"10.1007\/s10791-008-9059-7","volume":"11","author":"T Sakai","year":"2008","unstructured":"Sakai, T., & Kando, N. (2008). On information retrieval metrics designed for evaluation with incomplete relevance assessments. Information Retrieval, 11(5), 447\u2013470. https:\/\/doi.org\/10.1007\/s10791-008-9059-7.","journal-title":"Information Retrieval"},{"key":"9377_CR57","unstructured":"Schnabel, T., Swaminathan, A., Singh, A., Chandak, N., & Joachims, T. (2016). Recommendations as treatments: Debiasing learning and evaluation. In Proceedings of the 33rd international conference on machine learning, ICML \u201916 (pp. 1670\u20131679)."},{"key":"9377_CR58","volume-title":"A\/B testing: The most powerful way to turn clicks into customers","author":"D Siroker","year":"2013","unstructured":"Siroker, D., & Koomen, P. (2013). A\/B testing: The most powerful way to turn clicks into customers. New York: Wiley."},{"key":"9377_CR59","unstructured":"Sp\u00e4rck Jones, K., & Van Rijsbergen, C.J. (1975). Report on the need for and provision of an ideal information retrieval test collection. British Library Research and Development Reports. University Computer Laboratory"},{"key":"9377_CR60","doi-asserted-by":"publisher","unstructured":"Steck, H. (2010). Training and testing of recommender systems on data missing not at random. In Proceedings of the 16th ACM SIGKDD international conference on knowledge discovery and data mining, KDD \u201910 (pp. 713\u2013722). New York, NY: ACM. https:\/\/doi.org\/10.1145\/1835804.1835895.","DOI":"10.1145\/1835804.1835895"},{"key":"9377_CR61","unstructured":"Swaminathan, A., Krishnamurthy, A., Agarwal, A., Dud\u00edk, M., Langford, J., Jose, D., Zitouni, I. (2017). Off-policy evaluation for slate recommendation. In Proceedings of the 31st annual conference on neural information processing systems, NIPS \u201917 (pp. 3635\u20133645)."},{"key":"9377_CR62","doi-asserted-by":"publisher","first-page":"623","DOI":"10.1145\/1577069.1577091","volume":"10","author":"G Tak\u00e1cs","year":"2009","unstructured":"Tak\u00e1cs, G., Pil\u00e1szy, I., N\u00e9meth, B., & Tikk, D. (2009). Scalable collaborative filtering approaches for large recommender systems. Journal of Machine Learning Research, 10, 623\u2013656. https:\/\/doi.org\/10.1145\/1577069.1577091.","journal-title":"Journal of Machine Learning Research"},{"key":"9377_CR63","doi-asserted-by":"publisher","unstructured":"Valcarce, D., Bellog\u00edn, A., Parapar, J., & Castells, P. (2018). On the robustness and discriminative power of information retrieval metrics for top-N recommendation. In Proceedings of the 12th ACM conference on recommender systems, RecSys \u201918 (pp. 260\u2013268). New York, NY: ACM. https:\/\/doi.org\/10.1145\/3240323.3240347.","DOI":"10.1145\/3240323.3240347"},{"key":"9377_CR64","doi-asserted-by":"publisher","unstructured":"Valcarce, D., Parapar, J., & Barreiro, \u00c1. (2016). Efficient pseudo-relevance feedback methods for collaborative filtering recommendation. In Proceedings of the 38th European conference on information retrieval, ECIR \u201916 (pp. 602\u2013613). Berlin: Springer. https:\/\/doi.org\/10.1007\/978-3-319-30671-1_44.","DOI":"10.1007\/978-3-319-30671-1_44"},{"key":"9377_CR65","doi-asserted-by":"publisher","first-page":"41","DOI":"10.1016\/j.knosys.2016.03.021","volume":"103","author":"D Valcarce","year":"2016","unstructured":"Valcarce, D., Parapar, J., & Barreiro, \u00c1. (2016). Item-based relevance modelling of recommendations for getting rid of long tail products. Knowledge-Based Systems, 103, 41\u201351. https:\/\/doi.org\/10.1016\/j.knosys.2016.03.021.","journal-title":"Knowledge-Based Systems"},{"key":"9377_CR66","doi-asserted-by":"publisher","unstructured":"Valcarce, D., Parapar, J., Barreiro, \u00c1. (2016). Language models for collaborative filtering neighbourhoods. In Proceedings of the 38th European conference on information retrieval, ECIR \u201916 (pp. 614\u2013625). Berlin: Springer. https:\/\/doi.org\/10.1007\/978-3-319-30671-1_45.","DOI":"10.1007\/978-3-319-30671-1_45"},{"key":"9377_CR67","doi-asserted-by":"publisher","unstructured":"Voorhees, E.M. (2002). The philosophy of information retrieval evaluation. In Evaluation of cross-language information retrieval systems: Second workshop of the cross-language evaluation forum, CLEF 2001 (pp. 355\u2013370). Berlin: Springer. https:\/\/doi.org\/10.1007\/3-540-45691-0_34.","DOI":"10.1007\/3-540-45691-0_34"},{"key":"9377_CR68","doi-asserted-by":"crossref","unstructured":"Voorhees, E.M. (2005). Overview of the TREC 2004 Robust Track. In ACM SIGIR forum","DOI":"10.6028\/NIST.SP.500-261.robust-overview"},{"key":"9377_CR69","doi-asserted-by":"publisher","unstructured":"Wang, J., de Vries, A.P., & Reinders, M. J. T. (2006). A user-item relevance model for log-based collaborative filtering. In Proceedings of the 28th European conference on IR research, ECIR \u201906 (pp. 37\u201348). London: Springer. https:\/\/doi.org\/10.1007\/11735106_5.","DOI":"10.1007\/11735106_5"},{"key":"9377_CR70","unstructured":"Webber, W., Moffat, A., & Zobel, J. (2010). The effect of pooling and evaluation depth on metric stability. In Proceedings of the 3rd international workshop on evaluating information access, EVIA \u201910 (pp. 7\u201315)."},{"key":"9377_CR71","doi-asserted-by":"crossref","unstructured":"Yang, L., Cui, Y., Xuan, Y., Wang, C., Belongie, S., & Estrin, D. (2018). Unbiased offline recommender evaluation for missing-not-at-random implicit feedback. In Proceedings of the 12th ACM conference on recommender systems, RecSys \u201918 (pp. 279\u2013287). ACM.","DOI":"10.1145\/3240323.3240355"},{"issue":"2","key":"9377_CR72","doi-asserted-by":"publisher","first-page":"173","DOI":"10.1007\/s10115-007-0101-7","volume":"16","author":"E Yilmaz","year":"2008","unstructured":"Yilmaz, E., & Aslam, J. A. (2008). Estimating average precision when judgments are incomplete. Knowledge and Information Systems, 16(2), 173\u2013211. https:\/\/doi.org\/10.1007\/s10115-007-0101-7.","journal-title":"Knowledge and Information Systems"},{"issue":"9","key":"9377_CR73","doi-asserted-by":"publisher","first-page":"896","DOI":"10.14778\/2311906.2311916","volume":"5","author":"H Yin","year":"2012","unstructured":"Yin, H., Cui, B., Li, J., Yao, J., & Chen, C. (2012). Challenging the long tail recommendation. Proceedings of the VLDB Endowment, 5(9), 896\u2013907. https:\/\/doi.org\/10.14778\/2311906.2311916.","journal-title":"Proceedings of the VLDB Endowment"},{"issue":"4","key":"9377_CR74","doi-asserted-by":"publisher","first-page":"799","DOI":"10.1016\/j.ipm.2017.03.001","volume":"53","author":"HT Yu","year":"2017","unstructured":"Yu, H. T., Jatowt, A., Blanco, R., Joho, H., & Jose, J. M. (2017). An in-depth study on diversity evaluation: The importance of intrinsic diversity. Information Processing & Management, 53(4), 799\u2013813. https:\/\/doi.org\/10.1016\/j.ipm.2017.03.001.","journal-title":"Information Processing & Management"}],"container-title":["Information Retrieval Journal"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-020-09377-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10791-020-09377-x\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-020-09377-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,7]],"date-time":"2024-08-07T00:28:59Z","timestamp":1722990539000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10791-020-09377-x"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,6,8]]},"references-count":74,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2020,8]]}},"alternative-id":["9377"],"URL":"https:\/\/doi.org\/10.1007\/s10791-020-09377-x","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"value":"1386-4564","type":"print"},{"value":"1573-7659","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,6,8]]},"assertion":[{"value":"19 August 2019","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"20 May 2020","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"8 June 2020","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}