{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,8]],"date-time":"2026-10-08T15:15:17Z","timestamp":1791472517759,"version":"4.3.4"},"publisher-location":"Berlin, Heidelberg","reference-count":49,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"value":"9783642451348","type":"print"},{"value":"9783642451355","type":"electronic"}],"license":[{"start":{"date-parts":[[2013,12,20]],"date-time":"2013-12-20T00:00:00Z","timestamp":1387497600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2013,12,20]],"date-time":"2013-12-20T00:00:00Z","timestamp":1387497600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2014]]},"DOI":"10.1007\/978-3-642-45135-5_11","type":"book-chapter","created":{"date-parts":[[2014,4,30]],"date-time":"2014-04-30T09:11:18Z","timestamp":1398849078000},"page":"275-300","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["Benchmarking"],"prefix":"10.1007","author":[{"given":"Alan","family":"Said","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Domonkos","family":"Tikk","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Paolo","family":"Cremonesi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2013,12,20]]},"reference":[{"key":"11_CR1","doi-asserted-by":"crossref","unstructured":"Adomavicius, G., Zhang, J.: Stability of recommendation algorithms. ACM Trans. Inform. Syst. 30(4), 23:1\u201323:31 (2012). doi:10.1145\/2382438.2382442","DOI":"10.1145\/2382438.2382442"},{"key":"11_CR2","unstructured":"Amatriain, X., Basilico, J.: Netflix recommendations: Beyond the 5 stars (Part 1)\u2014The Netflix tech blog. URL http:\/\/techblog.netflix.com\/2012\/04\/netflix-recommendations-beyond-5-stars.html (2012) Accessed 9\u00a0October 2013"},{"key":"11_CR3","volume-title":"Recommendation Systems in Software Engineering, Chap. 10","author":"I. Avazpour","year":"2014","unstructured":"Avazpour, I., Pitakrat, T., Grunske, L., Grundy, J.: Dimensions and metrics for evaluating recommendation systems. In: Robillard, M., Maalej, W., Walker, R.J., Zimmermann, T. (eds.) Recommendation Systems in Software Engineering, Chap. 10. Springer, New York (2014)"},{"issue":"4\u20135","key":"11_CR4","doi-asserted-by":"publisher","first-page":"424","DOI":"10.1007\/s10664-010-9144-6","volume":"17","author":"S.K. Bajracharya","year":"2012","unstructured":"Bajracharya, S.K., Lopes, C.V.: Analyzing and mining a code search engine usage log. Empir. Software Eng. 17(4\u20135), 424\u2013466 (2012). doi:10.1007\/s10664-010-9144-6","journal-title":"Empir. Software Eng."},{"key":"11_CR5","unstructured":"Barber, W., Badre, A.: Culturability: The merging of culture and usability. In: Proceedings of the Conference on Human Factors & the Web, Basking Ridge, NJ, USA, 5 June 1998"},{"issue":"2","key":"11_CR6","first-page":"4","volume":"18","author":"R. Bell","year":"2007","unstructured":"Bell, R., Koren, Y., Volinsky, C.: Chasing $1,000,000: How we won the Netflix Progress Prize. ASA Stat. Comput. Graph. Newslett. 18(2), 4\u201312 (2007)","journal-title":"ASA Stat. Comput. Graph. Newslett."},{"key":"11_CR7","volume-title":"Benchmarking for Competitive Advantage","author":"R.J. Boxwell Jr","year":"1994","unstructured":"Boxwell Jr., R.J.: Benchmarking for Competitive Advantage. McGraw-Hill, New York (1994)"},{"key":"11_CR8","doi-asserted-by":"crossref","unstructured":"Butkiewicz, M., Madhyastha, H.V., Sekar, V.: Understanding website complexity: Measurements, metrics, and implications. In: Proceedings of the ACM SIGCOMM Conference on Internet Measurement, pp.\u00a0313\u2013328, Berlin, Germany, 2 November 2011. doi:10.1145\/2068816.2068846","DOI":"10.1145\/2068816.2068846"},{"key":"11_CR9","unstructured":"Carenini, G.: User-specific decision-theoretic accuracy metrics for collaborative filtering. In: Proceedings of the International Conference on Intelligent User Interfaces, San Diego, CA, USA, 10\u201313 January 2005"},{"key":"11_CR10","doi-asserted-by":"crossref","unstructured":"Celma, \u00d2., Lamere, P.: If you like the Beatles you might like \u2026: A tutorial on music recommendation. In: Proceedings of the ACM International Conference on Multimedia, pp.\u00a01157\u20131158. ACM, New York (2008). doi:10.1145\/1459359.1459615","DOI":"10.1145\/1459359.1459615"},{"key":"11_CR11","doi-asserted-by":"crossref","unstructured":"Chen, L., Pu, P.: A cross-cultural user evaluation of product recommender interfaces. In: Proceedings of the ACM Conference on Recommender Systems, pp. 75\u201382, Lousanne, Switzerland, 23\u201325 October 2008. doi:10.1145\/1454008.1454022","DOI":"10.1145\/1454008.1454022"},{"issue":"3","key":"11_CR12","doi-asserted-by":"publisher","first-page":"370","DOI":"10.1109\/TKDE.2007.48","volume":"19","author":"R.L. Cilibrasi","year":"2007","unstructured":"Cilibrasi, R.L., Vit\u00e1nyi, P.M.B.: The Google similarity distance. IEEE Trans. Knowl. Data Eng. 19(3), 370\u2013383 (2007). doi:10.1109\/TKDE.2007.48","journal-title":"IEEE Trans. Knowl. Data Eng."},{"key":"11_CR13","doi-asserted-by":"crossref","unstructured":"Cremonesi, P., Garzotto, F., Negro, S., Papadopoulos, A.V., Turrin, R.: Looking for \u201cgood\u201d recommendations: A comparative evaluation of recommender systems. In: Proceedings of the IFIP TC13 International Conference on Human\u2013Computer Interactaction, Part III, pp.\u00a0152\u2013168, Lisbon, Portugal, 5\u20139 September 2011. doi:10.1007\/978-3-642-23765-2_11","DOI":"10.1007\/978-3-642-23765-2_11"},{"key":"11_CR14","doi-asserted-by":"crossref","unstructured":"Cremonesi, P., Garzotto, F., Turrin, R.: Investigating the persuasion potential of recommender systems from a quality perspective: An empirical study. ACM Trans. Interact. Intell. Syst. 2(2), 11:1\u201311:41 (2012). doi:10.1145\/2209310.2209314","DOI":"10.1145\/2209310.2209314"},{"key":"11_CR15","doi-asserted-by":"publisher","first-page":"107","DOI":"10.1007\/978-0-387-85820-3_4","volume-title":"Recommender Systems Handbook","author":"C. Desrosiers","year":"2011","unstructured":"Desrosiers, C., Karypis, G.: A comprehensive survey of neighborhood-based recommendation methods. In: Ricci, F., Rokach, L., Shapira, B., Kantor, P.B. (eds.) Recommender Systems Handbook, pp.\u00a0107\u2013144. Springer, Boston (2011). doi:10.1007\/978-0-387-85820-3_4"},{"key":"11_CR16","doi-asserted-by":"crossref","unstructured":"Dias, M.B., Locher, D., Li, M., El-Deredy, W., Lisboa, P.J.G.: The value of personalised recommender systems to e-business: A case study. In: Proceedings of the ACM Conference on Recommender Systems, pp. 291\u2013294, Lousanne, Switzerland, 23\u201325 October 2008. doi:10.1145\/1454008.1454054","DOI":"10.1145\/1454008.1454054"},{"key":"11_CR17","doi-asserted-by":"publisher","DOI":"10.1007\/b101915","volume-title":"Multiple Criteria Optimization: State of the Art Annotated Bibliographic Surveys","year":"2002","unstructured":"Ehrgott, M., Gandibleux, X. (eds.): Multiple Criteria Optimization: State of the Art Annotated Bibliographic Surveys. Kluwer, Boston (2002). doi:10.1007\/b101915"},{"key":"11_CR18","doi-asserted-by":"crossref","unstructured":"Fraser, G., Arcuri, A.: Sound empirical evidence in software testing. In: Proceedings of the ACM\/IEEE International Conference on Software Engineering, pp.\u00a0178\u2013188, Zurich, Switzerland, 2\u20139 June 2012. doi:10.1109\/ICSE.2012.6227195","DOI":"10.1109\/ICSE.2012.6227195"},{"key":"11_CR19","doi-asserted-by":"crossref","unstructured":"Goh, D., Razikin, K., Lee, C.S., Chu, A.: Investigating user perceptions of engagement and information quality in mobile human computation games. In: Proceedings of the ACM\/IEEE-CS Joint Conference on Digital Libraries, pp.\u00a0391\u2013392, Washington, DC, USA, 10\u201314 June 2012. doi:10.1145\/2232817.2232906","DOI":"10.1145\/2232817.2232906"},{"key":"11_CR20","unstructured":"Gomez-Uribe, C.: Challenges and limitations in the offline and online evaluation of recommender systems: A Netflix case study. In: Proceedings of the Workshop on Recommendation Utility Evaluation: Beyond RMSE, CEUR Workshop Proceedings, vol.\u00a0910, p.\u00a01 (2012)"},{"key":"11_CR21","first-page":"2935","volume":"10","author":"A. Gunawardana","year":"2009","unstructured":"Gunawardana, A., Shani, G.: A survey of accuracy evaluation metrics of recommendation tasks. J. Mach. Learn. Res. 10, 2935\u20132962 (2009)","journal-title":"J. Mach. Learn. Res."},{"issue":"1","key":"11_CR22","doi-asserted-by":"publisher","first-page":"5","DOI":"10.1145\/963770.963772","volume":"22","author":"J.L. Herlocker","year":"2004","unstructured":"Herlocker, J.L., Konstan, J.A., Terveen, L.G., Riedl, J.T.: Evaluating collaborative filtering recommender systems. ACM Trans. Inform. Syst. 22(1), 5\u201353 (2004). doi:10.1145\/963770.963772","journal-title":"ACM Trans. Inform. Syst."},{"key":"11_CR23","doi-asserted-by":"crossref","unstructured":"Hu, R.: Design and user issues in personality-based recommender systems. In: Proceedings of the ACM Conference on Recommender Systems, pp. 357\u2013360, Barcelona, Spain, 26\u201330 Septembert 2010. doi:10.1145\/1864708.1864790","DOI":"10.1145\/1864708.1864790"},{"key":"11_CR24","doi-asserted-by":"crossref","unstructured":"Jambor, T., Wang, J.: Optimizing multiple objectives in collaborative filtering. In: Proceedings of the ACM Conference on Recommender Systems, pp. 55\u201362, Barcelona, Spain, 26\u201330 Septembert 2010. doi:10.1145\/1864708.1864723","DOI":"10.1145\/1864708.1864723"},{"key":"11_CR25","doi-asserted-by":"crossref","unstructured":"Koenigstein, N., Dror, G., Koren, Y.: Yahoo! music recommendations: Modeling music ratings with temporal dynamics and item taxonomy. In: Proceedings of the ACM Conference on Recommender Systems, pp. 165\u2013172, Chicago, IL, USA, 23\u201327 October 2011. doi:10.1145\/2043932.2043964","DOI":"10.1145\/2043932.2043964"},{"issue":"4","key":"11_CR26","doi-asserted-by":"publisher","first-page":"469","DOI":"10.1145\/321906.321910","volume":"22","author":"H.T. Kung","year":"1975","unstructured":"Kung, H.T., Luccio, F., Preparata, F.P.: On finding the maxima of a set of vectors. J. ACM 22(4), 469\u2013476 (1975). doi:10.1145\/321906.321910","journal-title":"J. ACM"},{"key":"11_CR27","doi-asserted-by":"crossref","unstructured":"Lai, J.Y.: Assessment of employees\u2019 perceptions of service quality and satisfaction with e-business. In: Proceedings of the ACM SIGMIS CPR Conference on Computer Personnel Research, pp.\u00a0236\u2013243, Claremont, CA, USA, 13\u201315 April 2006. doi:10.1145\/1125170.1125228","DOI":"10.1145\/1125170.1125228"},{"key":"11_CR28","doi-asserted-by":"crossref","unstructured":"Liu, J., Dolan, P., Pedersen, E.R.: Personalized news recommendation based on click behavior. In: Proceedings of the International Conference on Intelligent User Interfaces, pp.\u00a031\u201340, Hong Kong, China, 7\u201310 February 2010. doi:10.1145\/1719970.1719976","DOI":"10.1145\/1719970.1719976"},{"key":"11_CR29","unstructured":"McNee, S., Lam, S.K., Guetzlaff, C., Konstan, J.A., Riedl, J.: Confidence displays and training in recommender systems. In: Proceedings of the IFIP TC13 International Conference on Human\u2013Computer Interactaction, pp.\u00a0176\u2013183, Zurich, Switzerland, 1\u20135 September 2003"},{"issue":"3","key":"11_CR30","doi-asserted-by":"publisher","first-page":"153","DOI":"10.1080\/01449290410001669914","volume":"23","author":"F.F.H. Nah","year":"2004","unstructured":"Nah, F.F.H.: A study on tolerable waiting time: How long are Web users willing to wait? Behav. Inform. Technol. 23(3), 153\u2013163 (2004). doi:10.1080\/01449290410001669914","journal-title":"Behav. Inform. Technol."},{"key":"11_CR31","unstructured":"Netflix Prize: The Netflix Prize rules (2006). URL http:\/\/www.netflixprize.com\/rules. Accessed 9\u00a0October 2013"},{"key":"11_CR32","volume-title":"Enterprise content management: Expected evolution or vendor positioning?","author":"R. Perry","year":"2002","unstructured":"Perry, R., Lancaster, R.: Enterprise content management: Expected evolution or vendor positioning? Tech. rep., The Yankee Group (2002)"},{"key":"11_CR33","unstructured":"Pe\u0161ka, L., Vojt\u00e1\u0161, P.: Evaluating the importance of various implicit factors in E-commerce. In: Proceedings of the Workshop on Recommendation Utility Evaluation: Beyond RMSE, CEUR Workshop Proceedings, vol.\u00a0910, pp.\u00a051\u201355, Dublin, Ireland, 9 September 2012"},{"key":"11_CR34","doi-asserted-by":"crossref","unstructured":"Pil\u00e1szy, I., Tikk, D.: Recommending new movies: Even a few ratings are more valuable than metadata. In: Proceedings of the ACM Conference on Recommender Systems, pp. 93\u2013100, New York, NY, USA, 23\u201325 October 2009. doi:10.1145\/1639714.1639731","DOI":"10.1145\/1639714.1639731"},{"key":"11_CR35","doi-asserted-by":"crossref","unstructured":"Pu, P., Chen, L., Hu, R.: A user-centric evaluation framework for recommender systems. In: Proceedings of the ACM Conference on Recommender Systems, pp. 157\u2013164, Chicago, IL, USA, 23\u201327 October 2011. doi:10.1145\/2043932.2043962","DOI":"10.1145\/2043932.2043962"},{"key":"11_CR36","doi-asserted-by":"crossref","unstructured":"Said, A., Fields, B., Jain, B.J., Albayrak, S.: User-centric evaluation of a K-furthest neighbor collaborative filtering recommender algorithm. In: Proceedings of the ACM Conference on Computer Supported Cooperative Work, pp.\u00a01399\u20131408, San Antonio, TX, USA, 23\u201327 February 2013. doi:10.1145\/2441776.2441933","DOI":"10.1145\/2441776.2441933"},{"key":"11_CR37","unstructured":"Said, A., Tikk, D., Shi, Y., Larson, M., Stumpf, K., Cremonesi, P.: Recommender systems evaluation: A 3D benchmark. In: Proceedings of the Workshop on Recommendation Utility Evaluation: Beyond RMSE, CEUR Workshop Proceedings, vol.\u00a0910, pp.\u00a021\u201323, Dublin, Ireland, 9 September 2012"},{"key":"11_CR38","doi-asserted-by":"crossref","unstructured":"Sarwat, M., Bao, J., Eldawy, A., Levandoski, J.J., Magdy, A., Mokbel, M.F.: Sindbad: A\u00a0location-based social networking system. In: Proceedings of the ACM SIGMOD International Conference on Management of Data, pp.\u00a0649\u2013652, Scottsdale, AZ, USA, 20\u201324 May 2012. doi:10.1145\/2213836.2213923","DOI":"10.1145\/2213836.2213923"},{"key":"11_CR39","doi-asserted-by":"crossref","unstructured":"Schein, A.I., Popescul, A., Ungar, L.H., Pennock, D.M.: Methods and metrics for cold-start recommendations. In: Proceedings of the ACM SIGIR International Conference on Research and Development in Information Retrieval, pp.\u00a0253\u2013260, Tampere, Finland, 11\u201315 August 2002. doi:10.1145\/564376.564421","DOI":"10.1145\/564376.564421"},{"issue":"1","key":"11_CR40","doi-asserted-by":"publisher","first-page":"51","DOI":"10.1023\/B:ELEC.0000045973.51289.8c","volume":"5","author":"A.I. Schein","year":"2005","unstructured":"Schein, A.I., Popescul, A., Ungar, L.H., Pennock, D.M.: CROC: A new evaluation criterion for recommender systems. Electron. Commerce Res. 5(1), 51\u201374 (2005). doi:10.1023\/B:ELEC.0000045973.51289.8c","journal-title":"Electron. Commerce Res."},{"key":"11_CR41","doi-asserted-by":"crossref","unstructured":"Sch\u00fctze, H., Silverstein, C.: Projections for efficient document clustering. In: Proceedings of the ACM SIGIR International Conference on Research and Development in Information Retrieval, pp.\u00a074\u201381, Philadelphia, PA, USA, 27\u201331 July 1997. doi:10.1145\/258525.258539","DOI":"10.1145\/258525.258539"},{"key":"11_CR42","doi-asserted-by":"crossref","unstructured":"Sumner, T., Khoo, M., Recker, M., Marlino, M.: Understanding educator perceptions of \u201cquality\u201d in digital libraries. In: Proceedings of the ACM\/IEEE-CS Joint Conference on Digital Libraries, pp.\u00a0269\u2013279, Houston, Texas, USA, 27\u201331 May 2003. doi:10.1109\/JCDL.2003.1204876","DOI":"10.1109\/JCDL.2003.1204876"},{"key":"11_CR43","first-page":"623","volume":"10","author":"G. Tak\u00e1cs","year":"2009","unstructured":"Tak\u00e1cs, G., Pil\u00e1szy, I., N\u00e9meth, B., Tikk, D.: Scalable collaborative filtering approaches for large recommender systems. J. Mach. Learn. Res. 10, 623\u2013656 (2009)","journal-title":"J. Mach. Learn. Res."},{"key":"11_CR44","volume-title":"Human\u2013Computer Interaction in the New Millennium","author":"L. Terveen","year":"2001","unstructured":"Terveen, L., Hill, W.: Beyond recommender systems: Helping people help each other. In: Carroll, J.M. (ed.) Human\u2013Computer Interaction in the New Millennium. Addison-Wesley, New York (2001)"},{"issue":"2","key":"11_CR45","doi-asserted-by":"publisher","first-page":"125","DOI":"10.1162\/106365600568158","volume":"8","author":"D.A. Van Veldhuizen","year":"2000","unstructured":"Van Veldhuizen, D.A., Lamont, G.B.: Multiobjective evolutionary algorithms: Analyzing the state-of-the-art. Evol. Comput. 8(2), 125\u2013147 (2000). doi:10.1162\/106365600568158","journal-title":"Evol. Comput."},{"key":"11_CR46","doi-asserted-by":"crossref","unstructured":"Walker, R.J.: Recent advances in recommendation systems for software engineering. In: Proceedings of the International Conference on Industrial Engineering and Other Applications of Applied Intelligent Systems, Lecture Notes in Computer Science, vol.\u00a07906, pp.\u00a0372\u2013381. Springer, Heidelberg (2013). doi:10.1007\/978-3-642-38577-3_38","DOI":"10.1007\/978-3-642-38577-3_38"},{"key":"11_CR47","doi-asserted-by":"crossref","unstructured":"Zheng, H., Wang, D., Zhang, Q., Li, H., Yang, T.: Do clicks measure recommendation relevancy?: An empirical user study. In: Proceedings of the ACM Conference on Recommender Systems, pp. 249\u2013252, Barcelona, Spain, 26\u201330 September 2010. doi:10.1145\/1864708.1864759","DOI":"10.1145\/1864708.1864759"},{"key":"11_CR48","doi-asserted-by":"crossref","unstructured":"Ziegler, C.N., McNee, S.M., Konstan, J.A., Lausen, G.: Improving recommendation lists through topic diversification. In: Proceedings of the International Conference on the World Wide Web, pp.\u00a022\u201332, Chiba, Japan, 10\u201314 May 2005. doi:10.1145\/1060745.1060754","DOI":"10.1145\/1060745.1060754"},{"issue":"2","key":"11_CR49","doi-asserted-by":"publisher","first-page":"173","DOI":"10.1162\/106365600568202","volume":"8","author":"E. Zitzler","year":"2000","unstructured":"Zitzler, E., Deb, K., Thiele, L.: Comparison of multiobjective evolutionary algorithms: Empirical results. Evol. Comput. 8(2), 173\u2013195 (2000). doi:10.1162\/106365600568202","journal-title":"Evol. Comput."}],"container-title":["Recommendation Systems in Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-642-45135-5_11","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,30]],"date-time":"2023-01-30T23:40:28Z","timestamp":1675122028000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-642-45135-5_11"}},"subtitle":["A Methodology for Ensuring the Relative Quality of Recommendation Systems in Software Engineering"],"short-title":[],"issued":{"date-parts":[[2013,12,20]]},"ISBN":["9783642451348","9783642451355"],"references-count":49,"URL":"https:\/\/doi.org\/10.1007\/978-3-642-45135-5_11","relation":{},"subject":[],"published":{"date-parts":[[2013,12,20]]},"assertion":[{"value":"20 December 2013","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}}]}}