{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,14]],"date-time":"2026-04-14T00:07:59Z","timestamp":1776125279751,"version":"3.50.1"},"reference-count":42,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2017,12,14]],"date-time":"2017-12-14T00:00:00Z","timestamp":1513209600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61532011"],"award-info":[{"award-number":["61532011"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61672311"],"award-info":[{"award-number":["61672311"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"National Key Basic Research Program","award":["2015CB358700"],"award-info":[{"award-number":["2015CB358700"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Int. J. Mach. Learn. & Cyber."],"published-print":{"date-parts":[[2019,5]]},"DOI":"10.1007\/s13042-017-0762-9","type":"journal-article","created":{"date-parts":[[2017,12,14]],"date-time":"2017-12-14T13:07:07Z","timestamp":1513256827000},"page":"813-831","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":208,"title":["How good your recommender system is? A survey on evaluations in recommendation"],"prefix":"10.1007","volume":"10","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6854-1108","authenticated-orcid":false,"given":"Thiago","family":"Silveira","sequence":"first","affiliation":[]},{"given":"Min","family":"Zhang","sequence":"additional","affiliation":[]},{"given":"Xiao","family":"Lin","sequence":"additional","affiliation":[]},{"given":"Yiqun","family":"Liu","sequence":"additional","affiliation":[]},{"given":"Shaoping","family":"Ma","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2017,12,14]]},"reference":[{"key":"762_CR1","doi-asserted-by":"crossref","unstructured":"Adamopoulos P, Tuzhilin A (2014) On unexpectedness in recommender systems: or how to better expect the unexpected. ACM Trans Intell Syst Technol 5(4):Article 54","DOI":"10.1145\/2559952"},{"issue":"6","key":"762_CR2","doi-asserted-by":"publisher","first-page":"734749","DOI":"10.1109\/TKDE.2005.99","volume":"17","author":"G Adomavicius","year":"2005","unstructured":"Adomavicius G, Tuzhilin A (2005) Toward the next generation of recommender systems: a survey of the state-of-the-art and possible extensions. IEEE Trans on Knowl Data Eng 17(6):734\u2013749","journal-title":"IEEE Trans on Knowl Data Eng"},{"key":"762_CR3","unstructured":"Akiyama T, Obara K, Tanizaki M (2010) Proposal and evaluation of serendipitous recommendation method using general unexpectedness. In: Workshop on the practical use of recommender systems, algorithms and technologies"},{"key":"762_CR4","doi-asserted-by":"publisher","first-page":"109","DOI":"10.1016\/j.knosys.2013.03.012","volume":"46","author":"J Bobadilla","year":"2013","unstructured":"Bobadilla J, Ortega F, Hernando A, Guti\u00e9rrez A (2013) Recommender systems survey. Knowl Based Syst 46:109\u2013132","journal-title":"Knowl Based Syst"},{"key":"762_CR5","doi-asserted-by":"crossref","unstructured":"Celma O, Herrera P (2008) A new approach to evaluating novel recommendations. In: Proceedings of the 2008 ACM conference on recommender systems (RecSys \u201808). ACM, New York, NY, USA, pp 179\u2013186","DOI":"10.1145\/1454008.1454038"},{"key":"762_CR6","first-page":"87","volume-title":"The long tail in recommender systems","author":"O Celma","year":"2010","unstructured":"Celma O (2010) The long tail in recommender systems. Springer, Berlin, pp\u00a087\u2013107"},{"key":"762_CR7","doi-asserted-by":"crossref","unstructured":"Chu W, Park S (2009) Personalized recommendation on dynamic content using predictive bilinear models. In: Proceedings of the 18th international conference on World wide web (WWW \u201809). ACM, New York, NY, USA, pp 691\u2013700","DOI":"10.1145\/1526709.1526802"},{"key":"762_CR8","doi-asserted-by":"publisher","first-page":"2497","DOI":"10.1007\/s11042-013-1563-0","volume":"72","author":"T De Pessemier","year":"2014","unstructured":"De Pessemier T, Dooms S, Martens L (2014) Comparison of group recommendation algorithms. Multimed Tools Appl 72(3):2497\u20132541","journal-title":"Multimed Tools Appls"},{"key":"#cr-split#-762_CR9.1","unstructured":"Dror G, Koenigstein N, Koren Y, Weimer M (2011) The Yahoo! music dataset and KDD-Cup'11. In: Dror G, Koren Y, Weimer M"},{"key":"#cr-split#-762_CR9.2","unstructured":"(eds) Proceedings of the 2011 international conference on KDD Cup 2011-volume\u00a018 (KDDCUP'11), vol\u00a018. JMLR.org, pp 3-18"},{"key":"762_CR10","volume-title":"Marketing metrics: the definitive guide to measuring marketing performance","author":"PW Farris","year":"2010","unstructured":"Farris PW, Bendle NT, Pfeifer PE, Reibstein D (2010) Marketing metrics: the definitive guide to measuring marketing performance. Pearson Education, Inc., Upper Saddle River (ISBN 0\u201313-705829-2)"},{"issue":"5","key":"762_CR11","doi-asserted-by":"publisher","first-page":"695","DOI":"10.1016\/j.ipm.2015.06.008","volume":"51","author":"M Gemmis","year":"2015","unstructured":"Gemmis M, Lops P, Semeraro G, Musto C (2015) An investigation on the serendipity problem in recommender systems. Inf Process Manag 51(5):695\u2013717","journal-title":"Inf Process Manag"},{"key":"762_CR12","doi-asserted-by":"crossref","unstructured":"Ge M, Delgado-Battenfeld C, Jannach D (2010) Beyond accuracy: evaluating recommender systems by coverage and serendipity. In: Proceedings of the fourth ACM conference on recommender systems (RecSys \u201810). ACM, New York, NY, USA, pp 257\u2013260","DOI":"10.1145\/1864708.1864761"},{"issue":"4","key":"762_CR13","doi-asserted-by":"publisher","first-page":"13","DOI":"10.1145\/2843948","volume":"6","author":"C Gomez-Uribe","year":"2015","unstructured":"Gomez-Uribe C, Hunt N (2015) The Netflix recommender system: algorithms, business value, and innovation. ACM Trans Manag Inf Syst 6(4):Article 13","journal-title":"ACM Trans Manag Inf Syst"},{"issue":"1","key":"762_CR14","doi-asserted-by":"publisher","first-page":"553","DOI":"10.1145\/963770.963772","volume":"22","author":"J Herlocker","year":"2004","unstructured":"Herlocker J, Konstan J, Terveen L, Riedl J (2004) Evaluating collaborative filtering recommender systems. ACM Trans Inf Syst 22(1):5\u201353","journal-title":"ACM Trans Inf Syst"},{"key":"762_CR15","doi-asserted-by":"crossref","unstructured":"Hurley N, Zhang M (2011) Novelty and diversity in top-n recommendation\u2014analysis and evaluation. ACM Trans Internet Technol 10(4):Article 14","DOI":"10.1145\/1944339.1944341"},{"key":"762_CR16","doi-asserted-by":"crossref","unstructured":"Iaquinta L, Gemmis M, Lops P, Semeraro G. Molino P (2010) Can a recommender system induce serendipitous encounters? In: Kang K (ed) E-commerce. InTech. \n                    https:\/\/www.intechopen.com\/books\/e-commerce\/can-a-recommender-system-induce-serendipitous-encounters-","DOI":"10.5772\/8905"},{"key":"762_CR17","doi-asserted-by":"crossref","unstructured":"Iaquinta L, Gemmis M, Lops P, Semeraro G, Filannino M, Molino P (2008) Introducing serendipity in a content-based recommender system. In: Proceedings of the 2008 8th international conference on hybrid intelligent systems (HIS \u201808). IEEE Computer Society, Washington, DC, USA, pp 168\u2013173","DOI":"10.1109\/HIS.2008.25"},{"key":"762_CR18","unstructured":"Kaminskas M, Bridge D (2014) Measuring surprise in recommender systems. In: Workshop on recommender systems evaluation: dimensions and design (REDD 2014), October 10, 2014, Silicon Valley, USA"},{"key":"762_CR19","doi-asserted-by":"crossref","unstructured":"Kapoor K, Kumar V, Terveen L et al (2015) \u201cI Like to Explore Sometimes\u201d: adapting to dynamic user novelty preferences. In: Proceedings of the 9th ACM conference on recommender systems (RecSys \u201815). ACM, New York, NY, USA, pp 19\u201326","DOI":"10.1145\/2792838.2800172"},{"key":"762_CR20","doi-asserted-by":"crossref","unstructured":"Kotkov D, Veijalainen J, Wang S (2016) Challenges of serendipity in recommender systems. In: WEBIST 2016: proceedings of the 12th international conference on web information systems and technologies, vol\u00a02, pp\u00a0251\u2013256","DOI":"10.5220\/0005879802510256"},{"key":"762_CR21","doi-asserted-by":"publisher","first-page":"C180","DOI":"10.1016\/j.knosys.2016.08.014","volume":"111","author":"D Kotkov","year":"2016","unstructured":"Kotkov D, Wang S, Veijalainen J (2016) A survey of serendipity in recommender systems. Knowl Based Syst 111:180\u2013192","journal-title":"Knowl Based Syst"},{"key":"762_CR22","doi-asserted-by":"crossref","unstructured":"Lu Q, Chen T, Zhang W, Yang D, Yu Y (2012) Serendipitous personalized ranking for top-n recommendation. In: Proceedings of the 2012 IEEE\/WIC\/ACM international joint conferences on web intelligence and intelligent agent technology-volume\u00a001 (WI-IAT \u201812), vol\u00a01. IEEE Computer Society, Washington, DC, USA, pp 258\u2013265","DOI":"10.1109\/WI-IAT.2012.135"},{"key":"762_CR23","doi-asserted-by":"crossref","unstructured":"Maksai A, Garcin F, Faltings B (2015) Predicting online performance of news recommender systems through richer evaluation metrics. In: Proceedings of the 9th ACM conference on recommender systems (RecSys \u201815). ACM, New York, NY, USA, pp 179\u2013186","DOI":"10.1145\/2792838.2800184"},{"key":"762_CR24","doi-asserted-by":"crossref","unstructured":"McNee S, Riedl J, Konstan J (2006) Being accurate is not enough: how accuracy metrics have hurt recommender systems. In: CHI \u201806 extended abstracts on human factors in computing systems (CHI EA \u201806). ACM, New York, pp\u00a01097\u20131101","DOI":"10.1145\/1125451.1125659"},{"key":"762_CR25","doi-asserted-by":"crossref","unstructured":"Murakami T, Mori K, Orihara R (2008) Metrics for evaluating the serendipity of recommendation lists. In: Proceedings of the 2007 conference on new frontiers in artificial intelligence (JSAI\u201907). Springer, Berlin, Heidelberg, pp 40\u201346","DOI":"10.1007\/978-3-540-78197-4_5"},{"key":"762_CR26","unstructured":"Nakatsuji M, Fujiwara Y, Tanaka A et al (2010) Classical music for rock fans? Novel recommendations for expanding user interests. In: Proceedings of the 19th ACM international conference on Information and knowledge management (CIKM \u201810). ACM, New York, NY, USA, pp 949\u2013958"},{"key":"762_CR27","doi-asserted-by":"crossref","unstructured":"Onuma K, Tong H, Faloutsos C (2009) TANGENT: A novel, \u2018Surprise Me\u2019, recommendation algorithm. In: KDD\u201909. ACM, New York, NY, USA, pp\u00a0657\u2013666","DOI":"10.1145\/1557019.1557093"},{"key":"762_CR28","unstructured":"Ribeiro M, Ziviani N, De Moura E et al (2014) Multi objective pareto-efficient approaches for recommender systems. ACM Trans Intell Syst Technol 5(4):Article 53"},{"key":"762_CR29","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-387-85820-3","volume-title":"Recommender systems handbook","author":"F Ricci","year":"2011","unstructured":"Ricci F, Rokach L, Shapira B, Kantor P (2011) Recommender systems handbook. Springer, Berlin"},{"key":"762_CR30","unstructured":"Rosenfeld N, Mansour Y, Yom-Tov E (2016) Predicting Counterfactuals from large historical data and small randomized trials. arXiv preprint \n                    arXiv:1610.07667"},{"key":"762_CR31","volume-title":"The paradox of choice: why less is more","author":"B Schwartz","year":"2004","unstructured":"Schwartz B (2004) The paradox of choice: why less is more. Ecco, New York"},{"key":"762_CR32","doi-asserted-by":"crossref","unstructured":"Shani G, Gunawardana A (2009) Evaluating recommender systems. Microsoft Research. \n                    https:\/\/www.microsoft.com\/en-us\/research\/publication\/evaluating-recommender-systems\/","DOI":"10.1007\/978-0-387-85820-3_8"},{"key":"762_CR33","doi-asserted-by":"crossref","unstructured":"Shi XY, Luo X, Shang MS, Gu L (2017) Long-term performance of collaborative filtering based recommenders in temporally evolving systems. In: Neurocomputing, vol\u00a0267, pp\u00a0635\u2013643","DOI":"10.1016\/j.neucom.2017.06.026"},{"key":"762_CR34","doi-asserted-by":"crossref","unstructured":"Silveira T, Rocha L, Mour\u00e3o F, Gon\u00e7alves M (2017) A framework for unexpectedness evaluation in recommendation. In: Proceedings of the symposium on applied computing (SAC \u201817). ACM, New York, NY, USA, pp 1662\u20131667","DOI":"10.1145\/3019612.3019760"},{"key":"762_CR35","doi-asserted-by":"crossref","unstructured":"Vargas S, Baltrunas L, Karatzoglou A, Castells P (2014) Coverage, redundancy and size-awareness in genre diversity for recommender systems. In: RecSys\u201914. ACM, New York, pp\u00a0209\u2013216","DOI":"10.1145\/2645710.2645743"},{"key":"762_CR36","doi-asserted-by":"crossref","unstructured":"Vargas S, Castells P (2011) Rank and relevance in novelty and diversity metrics for recommender systems. In: RecSys\u201911. ACM, New York, NY, USA, pp\u00a0109\u2013116","DOI":"10.1145\/2043932.2043955"},{"key":"762_CR37","unstructured":"Wen Z (2008) Recommendation system based on collaborative filtering. \n                    http:\/\/www.zheng-wen.com\/WenRecommendation.pdf"},{"key":"762_CR38","doi-asserted-by":"crossref","unstructured":"Zhang Y, S\u00e9aghdha D, Quercia D, Jambor T (2012) Auralist: introducing serendipity into music recommendation. In: WSDM\u201912. ACM, New York, NY, USA, pp\u00a013\u201322","DOI":"10.1145\/2124295.2124300"},{"key":"762_CR39","doi-asserted-by":"crossref","unstructured":"Zhao Q, Chang S, Harper F, Konstan J (2016) Gaze prediction for recommender systems. In: RecSys\u201916. ACM, New York, pp\u00a0131\u2013138","DOI":"10.1145\/2959100.2959150"},{"issue":"10","key":"762_CR40","first-page":"45114515","volume":"107","author":"T Zhou","year":"2010","unstructured":"Zhou T, Kuscsik Z, Liu J et al (2010) Solving the apparent diversity-accuracy dilemma of recommender systems. Proc Natl Acad Sci 107(10):4511\u20134515","journal-title":"Proc Natl Acad Sci"},{"key":"762_CR41","doi-asserted-by":"crossref","unstructured":"Ziegler C, McNee S, Konstan J, Lausen G (2005) Improving recommendation lists through topic diversification. In: WWW\u201905. ACM, New York, NY, USA, pp\u00a022\u201332","DOI":"10.1145\/1060745.1060754"}],"container-title":["International Journal of Machine Learning and Cybernetics"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s13042-017-0762-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s13042-017-0762-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s13042-017-0762-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,4,24]],"date-time":"2019-04-24T05:40:21Z","timestamp":1556084421000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s13042-017-0762-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,12,14]]},"references-count":42,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2019,5]]}},"alternative-id":["762"],"URL":"https:\/\/doi.org\/10.1007\/s13042-017-0762-9","relation":{},"ISSN":["1868-8071","1868-808X"],"issn-type":[{"value":"1868-8071","type":"print"},{"value":"1868-808X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2017,12,14]]},"assertion":[{"value":"16 May 2017","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"5 December 2017","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"14 December 2017","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}