{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,3]],"date-time":"2025-11-03T23:00:02Z","timestamp":1762210802826},"reference-count":50,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2014,6,26]],"date-time":"2014-06-26T00:00:00Z","timestamp":1403740800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["User Model User-Adap Inter"],"published-print":{"date-parts":[[2015,3]]},"DOI":"10.1007\/s11257-014-9148-9","type":"journal-article","created":{"date-parts":[[2014,6,24]],"date-time":"2014-06-24T23:52:09Z","timestamp":1403653929000},"page":"1-37","update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":12,"title":["An automatic methodology to evaluate personalized information retrieval systems"],"prefix":"10.1007","volume":"25","author":[{"given":"Eduardo","family":"Vicente-L\u00f3pez","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Luis M.","family":"de Campos","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Juan M.","family":"Fern\u00e1ndez-Luna","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Juan F.","family":"Huete","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Antonio","family":"Tagua-Jim\u00e9nez","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Carmen","family":"Tur-Vigil","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2014,6,26]]},"reference":[{"key":"9148_CR1","doi-asserted-by":"crossref","first-page":"304","DOI":"10.1007\/3-540-48157-5_29","volume":"1707","author":"GD Abowd","year":"1999","unstructured":"Abowd, G.D., Dey, A.K., Brown, P.J., Davies, N., Smith, M., Steggles, P.: Towards a better understanding of context and context-awareness. Handheld Ubiquitous Comput. LNCS 1707, 304\u2013307 (1999)","journal-title":"Handheld Ubiquitous Comput. LNCS"},{"key":"9148_CR2","doi-asserted-by":"crossref","unstructured":"Allan, J.: Hard track overview in TREC 2003: High accuracy retrieval from documents. In: Proceedings of the 14th Text Retrieval Conference, Gaithersburg-Maryland, USA (2005)","DOI":"10.21236\/ADA439446"},{"key":"9148_CR3","doi-asserted-by":"crossref","unstructured":"Azzopardi, L., de Rijke, M., Balog, K.: Building simulated queries for known-item topics: an analysis using six european languages. In: Proceedings of the 30th ACM SIGIR Conference on Research and Development in Information Retrieval, Amsterdam, The Netherlands, pp. 455\u2013462 (2007)","DOI":"10.1145\/1277741.1277820"},{"key":"9148_CR4","doi-asserted-by":"crossref","unstructured":"Baiely, P., Craswell, N., Soboroff, I., Thomas, P., de Vries, A.P., Yilmaz, E.: Relevance assessment: are judges exchangeable and does it matter? In: Proceedings of the 31st ACM SIGIR Conference on Research and Development in Information Retrieval, Singapore, pp. 667\u2013674 (2008)","DOI":"10.1145\/1390334.1390447"},{"issue":"3","key":"9148_CR5","first-page":"152","volume":"8","author":"P Borlund","year":"2003","unstructured":"Borlund, P.: The IIR evaluation model: a framework for evaluation of interactive information retrieval systems. J. Inf. Res. 8(3), 152 (2003)","journal-title":"J. Inf. Res."},{"key":"9148_CR6","doi-asserted-by":"crossref","unstructured":"Buckley, C., Voorhees, E.M.: Retrieval evaluation with incomplete information. In: Proceedings of the 27th ACM SIGIR Conference on Research and Development in Information Retrieval, Sheffield, UK, pp. 25\u201332 (2004)","DOI":"10.1145\/1008992.1009000"},{"issue":"2","key":"9148_CR7","doi-asserted-by":"crossref","first-page":"191","DOI":"10.1016\/0306-4573(95)80035-R","volume":"31","author":"K Bystrom","year":"1995","unstructured":"Bystrom, K., Jarvelin, K.: Task complexity affects information seeking and use. Inf. Process. Manag. 31(2), 191\u2013213 (1995)","journal-title":"Inf. Process. Manag."},{"issue":"5","key":"9148_CR8","doi-asserted-by":"crossref","first-page":"829","DOI":"10.1016\/j.ipm.2004.04.014","volume":"40","author":"LM Campos de","year":"2004","unstructured":"de Campos, L.M., Fern\u00e1ndez-Luna, J.M., Huete, J.F.: Using context information in structured document retrieval: an approach using influence diagrams. Inf. Process. Manag. 40(5), 829\u2013847 (2004)","journal-title":"Inf. Process. Manag."},{"key":"9148_CR9","doi-asserted-by":"crossref","first-page":"215","DOI":"10.1007\/978-3-540-31865-1_16","volume":"3408","author":"LM Campos de","year":"2005","unstructured":"de Campos, L.M., Fern\u00e1ndez-Luna, J.M., Huete, J.F.: Improving the context-based influence diagram model for structured document retrieval: removing topological restrictions and adding new evaluation methods. Lect. Notes Comput. Sci. Adv. Inf. Retr. 3408, 215\u2013229 (2005)","journal-title":"Lect. Notes Comput. Sci. Adv. Inf. Retr."},{"key":"9148_CR10","unstructured":"de Campos, L.M., Fern\u00e1ndez-Luna, J.M., Huete, J.F., Romero, A.E.: Garnata: An information retrieval system for structured documents based on probabilistic graphical models. In: Proceedings of the 11th International Conference on Information Processing and Management of Uncertainty in Knowledge-Based Systems, Paris, France, pp. 1024\u20131031 (2006)"},{"key":"9148_CR11","doi-asserted-by":"crossref","first-page":"39","DOI":"10.1007\/978-3-642-03761-0_4","volume":"5631","author":"LM Campos de","year":"2009","unstructured":"de Campos, L.M., Fern\u00e1ndez-Luna, J.M., Huete, J.F., Mart\u00edn-Dancausa, C., Romero, A.E.: New utility models for the Garnata information retrieval system at INEX\u201908. Lect. Notes Comput. Sci. Adv. Focus. Retr. 5631, 39\u201345 (2009)","journal-title":"Lect. Notes Comput. Sci. Adv. Focus. Retr."},{"issue":"2","key":"9148_CR12","first-page":"156","volume":"43","author":"LM Campos de","year":"2009","unstructured":"de Campos, L.M., Fern\u00e1ndez-Luna, J.M., Huete, J.F., Mart\u00edn-Dancausa, C., Tagua-Jim\u00e9nez, A., Tur-Vigil, C.: An integrated system for managing the andalusian parliament\u2019s digital library. Prog. Electron. Lib. Inf. Syst. 43(2), 156\u2013174 (2009)","journal-title":"Prog. Electron. Lib. Inf. Syst."},{"key":"9148_CR13","doi-asserted-by":"crossref","unstructured":"de Campos, L.M., Fern\u00e1ndez-Luna, J.M., Huete, J.F., Vicente-L\u00f3pez, E.: XML search personalization strategies using query expansion, reranking and a search engine modification. In: Proceedings of the 28th ACM Symposium on Applied Computing, Coimbra, Portugal, pp. 872\u2013877 (2013)","DOI":"10.1145\/2480362.2480531"},{"key":"9148_CR14","doi-asserted-by":"crossref","unstructured":"de Campos, L.M., Fern\u00e1ndez-Luna, J.M., Huete, J.F., Vicente-L\u00f3pez, E.: Using personalization to improve XML retrieval. IEEE Trans. Knowl. Data Eng. 26(5), 1280\u20131292 (2014)","DOI":"10.1109\/TKDE.2013.75"},{"key":"9148_CR15","doi-asserted-by":"crossref","unstructured":"Carterette, B., Pavlu, V., Kanoulas, E., Aslam, J., Allan, J.: Evaluation over thousands of queries. In: Proceedings of the 31st ACM International Conference on Reseach and Developments in Information Retrieval, Singapore, pp. 651\u2013658 (2008)","DOI":"10.1145\/1390334.1390445"},{"key":"9148_CR16","doi-asserted-by":"crossref","unstructured":"Carterette, B., Soboroff, I.: The effect of Assessor Error on IR System Evaluation. In: Proceedings of the 33rd ACM International Conference on Reseach and Developments in Information Retrieval, Geneva, Switzerland, pp. 539\u2013546 (2010)","DOI":"10.1145\/1835449.1835540"},{"issue":"1\u20132","key":"9148_CR17","doi-asserted-by":"crossref","first-page":"181","DOI":"10.1023\/A:1011127315884","volume":"11","author":"DN Chin","year":"2001","unstructured":"Chin, D.N.: Empirical evaluation of user models and user-adapted systems. User Model. User-Adapt. Interact. 11(1\u20132), 181\u2013194 (2001)","journal-title":"User Model. User-Adapt. Interact."},{"key":"9148_CR18","unstructured":"Cleverdon, C.W., Mills, J., Keen, M.: Factors determining the performance of indexing systems, vol. 1\u2014design. ASLIB Cranfield Project. Technical Report (1966)"},{"key":"9148_CR19","unstructured":"Daoud, M., Tamine-Lechani, L., Boughanem, M.: A contextual evaluation protocol for a session-based personalized search. In: Proceedings of the 2nd Workshop on Contextual Information Access, Seeking and Retrieval Evaluation (CIRSE) in conjunction with the 32nd European Conference on Information Retrieval, Toulouse, France (2009)"},{"issue":"3","key":"9148_CR20","doi-asserted-by":"crossref","first-page":"1293","DOI":"10.1016\/j.ipm.2007.08.001","volume":"44","author":"A D\u00edaz","year":"2008","unstructured":"D\u00edaz, A., Garc\u00eda, A., Gerv\u00e1s, P.: User-centred versus system-centred evaluation of a personalization system. Inf. Process. Manag. 44(3), 1293\u20131307 (2008)","journal-title":"Inf. Process. Manag."},{"issue":"4","key":"9148_CR21","doi-asserted-by":"crossref","first-page":"494","DOI":"10.1002\/asi.20497","volume":"58","author":"C Ding","year":"2007","unstructured":"Ding, C., Patra, J.C.: User modeling for personalized Web search with self-organizing map. J. Am. Soc. Inf. Sci. Technol. 58(4), 494\u2013507 (2007)","journal-title":"J. Am. Soc. Inf. Sci. Technol."},{"key":"9148_CR22","doi-asserted-by":"crossref","unstructured":"Dou, Z., Song, R., Wen, J.R.: A large-scale evaluation and analysis of personalized search strategies. In: Proceedings of the 16th International Conference on World Wide Web, Banff, Canada, pp. 581\u2013590 (2007)","DOI":"10.1145\/1242572.1242651"},{"key":"9148_CR23","doi-asserted-by":"crossref","unstructured":"Elsweiler, D., Losada, D.E., Toucedo, J.C., Fernandez, R.T.: Seeding simulated queries with user-study data for personal search evaluation. In: Proceedings of the 34th international ACM SIGIR conference on Research and development in Information Retrieval, pp. 25\u201334 (2011)","DOI":"10.1145\/2009916.2009924"},{"key":"9148_CR24","doi-asserted-by":"crossref","first-page":"381","DOI":"10.1007\/s11257-012-9124-1","volume":"23","author":"MR Ghorab","year":"2013","unstructured":"Ghorab, M.R., Zhou, D., O\u2019Connor, A., Wade, V.: Personalised information retrieval: survey and classification. User Model. User Adapt. Interact. 23, 381\u2013443 (2013)","journal-title":"User Model. User Adapt. Interact."},{"key":"9148_CR25","doi-asserted-by":"crossref","unstructured":"Harman, D.: Overview of the fourth text retrieval conference (trec-4). In: Proceedings of the 4th Text Retrieval Conference, Gaithersburg-Maryland, USA (1995)","DOI":"10.6028\/NIST.SP.500-225"},{"issue":"1","key":"9148_CR26","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1108\/eb026960","volume":"52","author":"P Ingwersen","year":"1996","unstructured":"Ingwersen, P.: Cognitive perspectives of information retrieval interaction: elements of a cognitive IR theory. J. Doc. 52(1), 3\u201350 (1996)","journal-title":"J. Doc."},{"issue":"4","key":"9148_CR27","doi-asserted-by":"crossref","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K Jarvelin","year":"2002","unstructured":"Jarvelin, K., Kekalainen, J.: Cumulative gain-based evaluation of IR techniques. ACM Trans. Inf. Syst. 20(4), 422\u2013446 (2002)","journal-title":"ACM Trans. Inf. Syst."},{"issue":"1","key":"9148_CR28","doi-asserted-by":"crossref","first-page":"28","DOI":"10.1109\/TKDE.2004.1264820","volume":"16","author":"F Liu","year":"2004","unstructured":"Liu, F., Yu, C., Meng, W.: Personalized web search for improving retrieval effectiveness. IEEE Trans. Knowl. Data Eng. 16(1), 28\u201340 (2004)","journal-title":"IEEE Trans. Knowl. Data Eng."},{"issue":"2","key":"9148_CR29","doi-asserted-by":"crossref","first-page":"199","DOI":"10.1023\/A:1023932221048","volume":"6","author":"J Mostafa","year":"2003","unstructured":"Mostafa, J., Mukhopadhyay, S., Palakal, M.: Simulation studies of different dimensions of users\u2019 interests and their impact on user modeling and information filtering. Inf. Retr. 6(2), 199\u2013223 (2003)","journal-title":"Inf. Retr."},{"key":"9148_CR30","doi-asserted-by":"crossref","unstructured":"Nuray, R., Can, F.: Automatic ranking of retrieval systems in imperfect environments. In: Proceedings of the 26th ACM SIGIR Conference on Research and Development in Informaion Retrieval, Toronto, Canada, pp. 379\u2013380 (2003)","DOI":"10.1145\/860435.860510"},{"issue":"2","key":"9148_CR31","doi-asserted-by":"crossref","first-page":"375","DOI":"10.1002\/asi.21403","volume":"62","author":"S Pal","year":"2011","unstructured":"Pal, S., Mitra, M., Kamps, J.: Evaluation effort, reliability and reusability in XML retrieval. J. Am. Soc. Inf. Sci. Technol. 62(2), 375\u2013394 (2011)","journal-title":"J. Am. Soc. Inf. Sci. Technol."},{"issue":"1","key":"9148_CR32","doi-asserted-by":"crossref","first-page":"22","DOI":"10.1016\/j.ipm.2007.01.024","volume":"44","author":"D Petrelli","year":"2008","unstructured":"Petrelli, D.: On the role of user-centred evaluation in the advancement of interactive information retrieval. Inf. Process. Manag. 44(1), 22\u201338 (2008)","journal-title":"Inf. Process. Manag."},{"issue":"12","key":"9148_CR33","doi-asserted-by":"crossref","first-page":"165","DOI":"10.1016\/S0164-1212(03)00015-3","volume":"70","author":"V Ramesh","year":"2004","unstructured":"Ramesh, V., Glass, Robert L., Vessey, Iris: Research in computer science: an empirical study. J. Syst. Softw. 70(12), 165\u2013176 (2004)","journal-title":"J. Syst. Softw."},{"key":"9148_CR34","doi-asserted-by":"crossref","unstructured":"Sakai, T., Kando, N.: On information retrieval metrics designed for evaluation with incomplete relevance assessments. Inf. Retr. 11(5), 447\u2013470 (2008)","DOI":"10.1007\/s10791-008-9059-7"},{"key":"9148_CR35","doi-asserted-by":"crossref","unstructured":"Sanderson, M., Soboroff, I.: Problems with Kendall\u2019s tau. In: Proceedings of the 30th ACM SIGIR Conference on Research and Development in Information Retrieval. Amsterdam, The Netherlands, pp. 839\u2013840 (2007)","DOI":"10.1145\/1277741.1277935"},{"key":"9148_CR36","unstructured":"Santos, Jr, E., Zhao, Q., Nguyen, H., Wang, H.: Impacts of user modeling on personalization of information retrieval: an evaluation with human intelligence analysts. In: Proceedings of the 4th Workshop on the Evaluation of Adaptive Systems (Held in Conjunction with the 10th International Conference on User Modeling). Edinburgh, UK, pp. 27\u201336 (2005)"},{"issue":"6","key":"9148_CR37","doi-asserted-by":"crossref","first-page":"321","DOI":"10.1002\/asi.4630260604","volume":"26","author":"T Saracevic","year":"1975","unstructured":"Saracevic, T.: Relevance: a review of and a framework for thinking on the notion in information science. J. Am. Soc. Inf. Sci. 26(6), 321\u2013343 (1975)","journal-title":"J. Am. Soc. Inf. Sci."},{"key":"9148_CR38","doi-asserted-by":"crossref","unstructured":"Sieg, A., Mobasher, B., Burke, R.: Web search personalization with ontological user profiles. In: Proceedings of the 16th ACM International Conference on Information and Knowledge Management, Lisbon, Portugal, pp. 525\u2013534 (2007)","DOI":"10.1145\/1321440.1321515"},{"key":"9148_CR39","doi-asserted-by":"crossref","unstructured":"Soboroff, I., Nicholas, C., Cahan, P.: Ranking retrieval systems without relevance judgments. In: Proceedings of the 24th ACM SIGIR Conference on Research and Development in Information Retrieval, New Orleans, USA, pp. 66\u201373 (2001)","DOI":"10.1145\/383952.383961"},{"key":"9148_CR40","doi-asserted-by":"crossref","unstructured":"Spink, A., Ozmutlu, S., Ozmutlu, H.C., Jansen, B.J.: US versus european web searching trends. In: Proceedings of the 25th ACM SIGIR Conference on Research and Development in Information Retrieval, Tampere, Finland, pp. 32\u201338 (2002)","DOI":"10.1145\/792550.792555"},{"issue":"4","key":"9148_CR41","doi-asserted-by":"crossref","first-page":"698","DOI":"10.1016\/j.ipm.2011.12.004","volume":"48","author":"B Steichen","year":"2012","unstructured":"Steichen, B., Ashman, H., Wade, V.: A comparative survey of personalised information retrieval and adaptive hypermedia techniques. Inf. Process. Manag. 48(4), 698\u2013724 (2012)","journal-title":"Inf. Process. Manag."},{"key":"9148_CR42","doi-asserted-by":"crossref","unstructured":"Sugiyama, K., Hatano, K., Yoshikawa, M.: Adaptive web search based on user profile constructed without any effort from users. In: Proceedings of the 13th International Conference on World Wide Web, Manhattan, NY, USA pp. 675\u2013684 (2004)","DOI":"10.1145\/988672.988764"},{"issue":"1","key":"9148_CR43","doi-asserted-by":"crossref","first-page":"162","DOI":"10.1016\/j.csi.2011.07.001","volume":"34","author":"M Taghavi","year":"2012","unstructured":"Taghavi, M., Patel, A., Schmidt, N., Wills, C., Tew, Y.: An analysis of web proxy logs with query distribution pattern approach for search engines. Comput. Stand. Interfaces 34(1), 162\u2013170 (2012)","journal-title":"Comput. Stand. Interfaces"},{"issue":"1","key":"9148_CR44","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1007\/s10115-009-0231-1","volume":"24","author":"L Tamine-Lechani","year":"2009","unstructured":"Tamine-Lechani, L., Boughanem, M., Daoud, M.: Evaluation of contextual information retrieval effectiveness: overview of issues and research. Knowl. Inf. Syst. 24(1), 1\u201334 (2009)","journal-title":"Knowl. Inf. Syst."},{"issue":"4","key":"9148_CR45","doi-asserted-by":"crossref","first-page":"496","DOI":"10.1109\/TKDE.2010.145","volume":"23","author":"X Tao","year":"2011","unstructured":"Tao, X., Li, Y., Zhong, N.: A personalized ontology model for web information gathering. IEEE Trans. Knowl. Data Eng. 23(4), 496\u2013511 (2011)","journal-title":"IEEE Trans. Knowl. Data Eng."},{"key":"9148_CR46","doi-asserted-by":"crossref","unstructured":"Turpin, A.H., William, H.: Why batch and user evaluations do not give the same results. In: Proceedings of the 24th ACM SIGIR Conference on Research and Development in Information Retrieval, New Orleans, USA, pp. 225\u2013231 (2001)","DOI":"10.1145\/383952.383992"},{"issue":"3","key":"9148_CR47","doi-asserted-by":"crossref","first-page":"325","DOI":"10.1145\/1080343.1080347","volume":"23","author":"RW White","year":"2005","unstructured":"White, R.W., Ruthven, I., Jose, J.M., Van Rijsbergen, C.J.: Evaluating implicit feedback models using searcher simulations. ACM Trans. Inf. Syst. 23(3), 325\u2013361 (2005)","journal-title":"ACM Trans. Inf. Syst."},{"key":"9148_CR48","unstructured":"White, R.W.: Contextual simulations for information retrieval evaluation. In: Proceedings of the 2nd ACM SIGIR Workshop on Information Retrieval in Context, Sheffield, UK, pp. 27\u201328 (2005)"},{"issue":"2","key":"9148_CR49","first-page":"112","volume":"6","author":"Y Yang","year":"2005","unstructured":"Yang, Y., Padmanabhan, B.: Evaluation of online personalization systems: a survey of evaluation schemes and a knowledge-based approach. J. Electron. Commer. Res. 6(2), 112\u2013122 (2005)","journal-title":"J. Electron. Commer. Res."},{"key":"9148_CR50","doi-asserted-by":"crossref","unstructured":"Zobel, J.: How reliable are the results of large-scale information retrieval experiments? In: Proceedings of the 21st ACM SIGIR Conference on Research and Development in Information Retrieval, Melbourne, Australia, pp. 307\u2013314 (1998)","DOI":"10.1145\/290941.291014"}],"container-title":["User Modeling and User-Adapted Interaction"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s11257-014-9148-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s11257-014-9148-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s11257-014-9148-9","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,8,11]],"date-time":"2019-08-11T18:04:26Z","timestamp":1565546666000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s11257-014-9148-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014,6,26]]},"references-count":50,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2015,3]]}},"alternative-id":["9148"],"URL":"https:\/\/doi.org\/10.1007\/s11257-014-9148-9","relation":{},"ISSN":["0924-1868","1573-1391"],"issn-type":[{"value":"0924-1868","type":"print"},{"value":"1573-1391","type":"electronic"}],"subject":[],"published":{"date-parts":[[2014,6,26]]}}}