{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T06:20:14Z","timestamp":1775283614488,"version":"3.50.1"},"reference-count":90,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2010,6,1]],"date-time":"2010-06-01T00:00:00Z","timestamp":1275350400000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Int J Digit Libr"],"published-print":{"date-parts":[[2010,6]]},"DOI":"10.1007\/s00799-011-0070-z","type":"journal-article","created":{"date-parts":[[2011,9,27]],"date-time":"2011-09-27T09:59:29Z","timestamp":1317117569000},"page":"91-109","source":"Crossref","is-referenced-by-count":10,"title":["On the evaluation of Geographic Information Retrieval systems"],"prefix":"10.1007","volume":"11","author":[{"given":"Damien","family":"Palacio","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Guillaume","family":"Cabanac","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Christian","family":"Sallaberry","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Gilles","family":"Hubert","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2010,6,1]]},"reference":[{"issue":"6","key":"70_CR1","doi-asserted-by":"publisher","first-page":"919","DOI":"10.1108\/02640470911004039","volume":"27","author":"J. Purday","year":"2009","unstructured":"Purday J.: Think culture: Europeana.eu from concept to construction. Electron. Libr. 27(6), 919\u2013937 (2009). doi: https:\/\/doi.org\/10.1108\/02640470911004039","journal-title":"Electron. Libr."},{"issue":"7","key":"70_CR2","first-page":"833","volume":"62","author":"E.L. Usery","year":"1996","unstructured":"Usery E.L.: A feature-based geographic information system model. Photogramm. Eng. Rem. Sens. 62(7), 833\u2013838 (1996) ISSN 0099-1112","journal-title":"Photogramm. Eng. Rem. Sens."},{"key":"70_CR3","unstructured":"Sanderson, M., Kohler, J.: Analyzing geographic queries. In: SIGIR-GIR\u201904: Proceedings of the Workshop on Geographic Information Retrieval at SIGIR (2004)"},{"key":"70_CR4","doi-asserted-by":"publisher","unstructured":"Asadi, S., Chang, C.-Y., Zhou, X., Diederich, J.: Searching the World Wide Web for local services and facilities: a review on the patterns of location-based queries. In: Fan, W., Wu, Z., Yang, J. (eds.) WAIM\u201905: Proceedings of the 6th International Conference on Web-Age Information Management, LNCS, vol. 3739, pp. 91\u2013101. Springer (2005). doi: https:\/\/doi.org\/10.1007\/11563952_9","DOI":"10.1007\/11563952_9"},{"key":"70_CR5","doi-asserted-by":"publisher","unstructured":"Souza, L.A., Davis, C.A., Jr., Borges, K.A.V., Delboni, T.M., Laender, A.H.F.: The role of gazetteers in geographic knowledge discovery on the Web. In: LA-WEB\u201905: Proceedings of the 3rd Latin American Web Congress, pp. 157\u2013165. IEEE Computer Society, October 2005. doi: https:\/\/doi.org\/10.1109\/LAWEB.2005.38","DOI":"10.1109\/LAWEB.2005.38"},{"key":"70_CR6","doi-asserted-by":"publisher","unstructured":"Gan, Q., Attenberg, J., Markowetz, A., Suel, T.: Analysis of geographic queries in a search engine log. In: LocWeb\u201908: Proceedings of the First International Workshop on Location and the Web, pp. 49\u201356. New York, NY, USA, 2008. ACM. doi: https:\/\/doi.org\/10.1145\/1367798.1367806","DOI":"10.1145\/1367798.1367806"},{"issue":"3","key":"70_CR7","doi-asserted-by":"publisher","first-page":"229","DOI":"10.1080\/13658810701626186","volume":"22","author":"R. Jones","year":"2008","unstructured":"Jones R., Zhang W.V., Rey B., Jhala P., Stipp E.: Geographic intention and modification in web search. Int. J. Geogr. Inf. Sci. 22(3), 229\u2013246 (2008). doi: https:\/\/doi.org\/10.1080\/13658810701626186","journal-title":"Int. J. Geogr. Inf. Sci."},{"issue":"1","key":"70_CR8","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1016\/j.jvlc.2007.08.010","volume":"19","author":"M. Gaio","year":"2008","unstructured":"Gaio M., Sallaberry C., Etcheverry P., Marquesuzaa C., Lesbegueries J.: A global process to access documents\u2019 contents from a geographical point of view. J. Vis. Lang. Comput. 19(1), 3\u201323 (2008). doi: https:\/\/doi.org\/10.1016\/j.jvlc.2007.08.010","journal-title":"J. Vis. Lang. Comput."},{"issue":"2","key":"70_CR9","doi-asserted-by":"publisher","first-page":"129","DOI":"10.3166\/dn.10.2.129-148","volume":"10","author":"A. LeParc-Lacayrelle","year":"2007","unstructured":"LeParc-Lacayrelle A., Gaio M., Sallaberry C.: La composante temps dans l\u2019information g\u00e9ographique textuelle. Document Num\u00e9rique 10(2), 129\u2013148 (2007). doi: https:\/\/doi.org\/10.3166\/dn.10.2.129-148","journal-title":"Document Num\u00e9rique"},{"key":"70_CR10","unstructured":"Sallaberry, C., Baziz, M., Lesbegueries, J., Gaio, M.: Towards an IE and IR system dealing with spatial information in digital libraries\u2014evaluation case study. In: Cardoso, J., Cordeiro, J., Filipe, J. (eds.) ICEIS\u201907: Proceedings of the 9th International Conference on Enterprise Information Systems, pp. 190\u2013197, 2007"},{"key":"70_CR11","doi-asserted-by":"crossref","unstructured":"Manning, C.D., Raghavan, P., Sch\u00fctze, H.: Introduction to information retrieval. Cambridge University Press, July 2008","DOI":"10.1017\/CBO9780511809071"},{"issue":"2","key":"70_CR12","doi-asserted-by":"publisher","first-page":"328","DOI":"10.1162\/coli.2003.29.2.328","volume":"29","author":"R.J. Gaizauskas","year":"2003","unstructured":"Gaizauskas R.J.: Recent advances in computational terminology edited by Didier Bourigault, Christian Jacquemin, and Marie-Claude L\u2019Homme. Comput. Linguist. 29(2), 328\u2013332 (2003). doi: https:\/\/doi.org\/10.1162\/coli.2003.29.2.328","journal-title":"Comput. Linguist."},{"key":"70_CR13","doi-asserted-by":"publisher","unstructured":"Cunningham, H., Maynard, D., Bontcheva, K., Tablan, V.: GATE: an architecture for development of robust HLT applications. In: ACL\u201902: Proceedings of the 40th Annual Meeting on Association for Computational Linguistics, pp. 168\u2013175, Morristown, NJ, USA, 2002. ACL. doi: https:\/\/doi.org\/10.3115\/1073083.1073112","DOI":"10.3115\/1073083.1073112"},{"key":"70_CR14","unstructured":"Bilhaut, F., Widl\u00f6cher, A.: Linguastream: an integrated environment for computational linguistics experimentation. In: EACL\u201906: Proceedings of the 11th Conference of the European Chapter of the Association for Computational Linguistics: Posters & Demonstrations, pp. 95\u201398, Morristown, NJ, USA, 2006. ACL. doi: 10.1.1.134.4336"},{"key":"70_CR15","doi-asserted-by":"publisher","unstructured":"Liu, Z., Gibbon, D.C., Shahraray, B.: Multimedia content acquisition and processing in the MIRACLE system. In: CCNC\u201906: Proceedings of the 3rd IEEE Conference on Consumer Communications and Networking, pp. 272\u2013276, January 2006. doi: https:\/\/doi.org\/10.1109\/CCNC.2006.1593030","DOI":"10.1109\/CCNC.2006.1593030"},{"issue":"2","key":"70_CR16","first-page":"155","volume":"49","author":"B. Sagot","year":"2008","unstructured":"Sagot B., Boullier P.: SxPipe 2: architecture pour le traitement pr\u00e9 syntaxique de corpus bruts. Traitement Automatique des Langues 49(2), 155\u2013188 (2008)","journal-title":"Traitement Automatique des Langues"},{"issue":"3\u20134","key":"70_CR17","doi-asserted-by":"publisher","first-page":"327","DOI":"10.1017\/S1351324904003523","volume":"10","author":"D. Ferruci","year":"2004","unstructured":"Ferruci D., Lally A.: UIMA: an architectural approach to unstructured information processing in the corporate research environment. Nat. Lang. Eng., 10(3\u20134), 327\u2013348 (2004). doi: https:\/\/doi.org\/10.1017\/S1351324904003523","journal-title":"Nat. Lang. Eng."},{"key":"70_CR18","doi-asserted-by":"publisher","unstructured":"Sp\u00e4rck Jones, K.S.: A statistical interpretation of term specificity and its application in retrieval. J. Doc., 28(1), 11\u201321 (1972). ISSN 0022-0418. doi: https:\/\/doi.org\/10.1108\/eb026526","DOI":"10.1108\/eb026526"},{"issue":"3","key":"70_CR19","doi-asserted-by":"publisher","first-page":"129","DOI":"10.1002\/asi.4630270302","volume":"27","author":"S.E. Robertson","year":"1976","unstructured":"Robertson S.E., Sp\u00e4rck Jones K.S.: Relevance weighting of search terms. J. Am. Soc. Inf. Sci. 27(3), 129\u2013146 (1976). doi: https:\/\/doi.org\/10.1002\/asi.4630270302","journal-title":"J. Am. Soc. Inf. Sci."},{"issue":"11","key":"70_CR20","doi-asserted-by":"publisher","first-page":"613","DOI":"10.1145\/361219.361220","volume":"18","author":"G. Salton","year":"1975","unstructured":"Salton G., Wong A., Yang C.S.: A vector space model for automatic indexing. Commun. ACM 18(11), 613\u2013620 (1975). doi: https:\/\/doi.org\/10.1145\/361219.361220","journal-title":"Commun. ACM"},{"key":"70_CR21","doi-asserted-by":"publisher","unstructured":"Larson, R.R., Frontiera, P.: Spatial ranking methods for geographic information retrieval (GIR) in digital libraries. In: ECDL\u201904: Proceedings of the 8th European Conference on Digital Libraries, LNCS, 3232, pp. 45\u201356. Springer, 2004. ISBN 3-540-23013-0. doi: https:\/\/doi.org\/10.1007\/978-3-540-30230-8_5","DOI":"10.1007\/978-3-540-30230-8_5"},{"key":"70_CR22","unstructured":"Andogah, G.: Geographically Constrained Information Retrieval. PhD thesis, University of Groningen, Netherlands, May 2010"},{"key":"70_CR23","unstructured":"Alonso, O.R.: Temporal information retrieval. PhD thesis, University of California, USA, May 2008"},{"issue":"3","key":"70_CR24","doi-asserted-by":"publisher","first-page":"635","DOI":"10.1016\/j.ipm.2004.01.002","volume":"41","author":"P.J. Kalczynski","year":"2005","unstructured":"Kalczynski, P.J., Chou, A.: Temporal document retrieval model for business news archives. Inf. Process. Manage, 41(3), 635\u2013650, 2005. ISSN 0306-4573. doi: https:\/\/doi.org\/10.1016\/j.ipm.2004.01.002","journal-title":"Inf. Process. Manage"},{"key":"70_CR25","doi-asserted-by":"publisher","unstructured":"Woodruff, A.G., Plaunt, C.: Gipsy: automated geographic indexing of text documents. J. Am. Soc. Inf. Sci., 45(9): 645\u2013655, 1994. ISSN 0002-8231. doi: https:\/\/doi.org\/10.1002\/(SICI)1097-4571(199410)45:9<645::AID-ASI2>3.0.CO;2-8","DOI":"10.1002\/(SICI)1097-4571(199410)45:9%3c645::AID-ASI2%3e3.0.CO;2-8"},{"key":"70_CR26","doi-asserted-by":"publisher","unstructured":"Bilhaut, F., Charnois, T., Enjalbert, P., Mathet, Y.: Geographic reference analysis for geographic document querying. In: HLT- NAACL\u201903: Proceedings of the Workshop on Analysis of Geographic References, pp. 55\u201362, Morristown, NJ, USA, 2003. ACL. doi: https:\/\/doi.org\/10.3115\/1119394.1119403","DOI":"10.3115\/1119394.1119403"},{"key":"70_CR27","doi-asserted-by":"publisher","unstructured":"Vaid, S., Jones, C.B., Joho, H., Sanderson, M.: Spatio-textual indexing for geographical search on the Web. In: SSTD\u201905: Proceedings of the 9th International Symposium on Spatial and Temporal Databases, LNCS, vol. 3633, pp. 218\u2013235. Springer, 2005. doi: https:\/\/doi.org\/10.1007\/11535331_13","DOI":"10.1007\/11535331_13"},{"key":"70_CR28","unstructured":"Valcartier. GRID\u2014geospatial retrieval of indexed document. Technical report, R&D pour la d\u00e9fense, Canada (2006)"},{"key":"70_CR29","doi-asserted-by":"publisher","unstructured":"Martins, B., Borbinha, J., Pedrosa, G., Gil, J., Freire, N.: Geographically-aware information retrieval for collections of digitized historical maps. In: GIR\u201907: Proceedings of the 4th ACM workshop on Geographical Information Retrieval, pp. 39\u201342, New York, NY, USA, 2007. ACM. doi: https:\/\/doi.org\/10.1145\/1316948.1316959","DOI":"10.1145\/1316948.1316959"},{"key":"70_CR30","doi-asserted-by":"publisher","unstructured":"Chen, Y.-F.R., Di Fabbrizio, G., Gibbon, D., Jora, S., Renger, B., Wei, B.: Geotracker: geospatial and temporal RSS navigation. In: WWW\u201907: Proceedings of the 16th International Conference on World Wide Web, pp. 41\u201350, New York, NY, USA, 2007. ACM. doi: https:\/\/doi.org\/10.1145\/1242572.1242579","DOI":"10.1145\/1242572.1242579"},{"key":"70_CR31","doi-asserted-by":"publisher","unstructured":"Lieberman, M.D., Samet, H., Sankaranarayanan, J., Sperling, J.: STEWARD: architecture of a Spatio-Textual search engine. In: GIS\u201907: Proceedings of the 15th Annual ACM International Symposium on Advances in Geographic Information Systems, pp. 1\u20138, New York, NY, USA, 2007. ACM. doi: https:\/\/doi.org\/10.1145\/1341012.1341045","DOI":"10.1145\/1341012.1341045"},{"key":"70_CR32","doi-asserted-by":"publisher","unstructured":"Pfoser, D., Efentakis, A., Hadzilacos, T., Karagiorgou, S., Vasiliou, G.: Providing universal access to history textbooks: a modified GIS case. In: W2GIS\u201909: Proceedings of the 9th International Symposium on Web and Wireless Geographical Information Systems, LNCS, vol. 5886, pp. 87\u2013102, 2009. doi: https:\/\/doi.org\/10.1007\/978-3-642-10601-9_7","DOI":"10.1007\/978-3-642-10601-9_7"},{"key":"70_CR33","unstructured":"Buscaldi, D., Rosso, P.: Geooreka: enhancing web searches with geographical information. In: De Antonellis, V., Castano, S., Catania, B., Guerrini, G. (eds.) SEBD\u201909: Proceedings of the Seventeenth Italian Symposium on Advanced Database Systems, pp. 205\u2013212. Edizioni Seneca, 2009"},{"key":"70_CR34","doi-asserted-by":"publisher","unstructured":"Garc\u00eda-Cumbreras, M.\u00c1., Perea-Ortega, J.M., Garc\u00eda-Vega, M., Ure\u00f1a-L\u00f3pez, L.A.: Information retrieval with geographical references. Relevant documents filtering vs. query expansion. Inf. Process. Manage. 45(5), 605\u2013614 (2009). ISSN 0306-4573. doi: https:\/\/doi.org\/10.1016\/j.ipm.2009.04.006","DOI":"10.1016\/j.ipm.2009.04.006"},{"key":"70_CR35","doi-asserted-by":"publisher","unstructured":"Str\u00f6tgen, J., Gertz, M., Popov, P.: Extraction and exploration of spatio-temporal information in documents. In: GIR\u201910: Proceedings of the 6th Workshop on Geographic Information Retrieval, pp. 16:1\u201316:8, New York, NY, USA, 2010. ACM. doi: https:\/\/doi.org\/10.1145\/1722080.1722101","DOI":"10.1145\/1722080.1722101"},{"issue":"3","key":"70_CR36","doi-asserted-by":"publisher","first-page":"307","DOI":"10.1007\/s10707-010-0106-3","volume":"14","author":"N. Brisaboa","year":"2010","unstructured":"Brisaboa N., Luaces M., Places , \u00c1. , Seco D.: Exploiting geographic references of documents in a geographical information retrieval system using an ontology-based index. GeoInformatica 14(3), 307\u2013331 (2010). doi: https:\/\/doi.org\/10.1007\/s10707-010-0106-3","journal-title":"GeoInformatica"},{"key":"70_CR37","doi-asserted-by":"publisher","unstructured":"Stokes, N., Li, Y., Moffat, A., Rong, J. An empirical study of the effects of NLP components on Geographic IR performance. Int. J. Geogr. Inf. Sci. 22(3), 247\u2013264, 2008. ISSN 1365-8816. doi: https:\/\/doi.org\/10.1080\/13658810701626210","DOI":"10.1080\/13658810701626210"},{"key":"70_CR38","doi-asserted-by":"crossref","unstructured":"Ounis, I., Amati, G., Plachouras, V., He, B., Macdonald, C., Lioma, C.: Terrier: a high performance and scalable information retrieval platform. In: OSIR\u201906: Proceedings of ACM SIGIR\u201906 Workshop on Open Source Information Retrieval (2006)","DOI":"10.1007\/978-3-540-31865-1_37"},{"key":"70_CR39","doi-asserted-by":"publisher","unstructured":"Clough, P., Joho, H., Purves, R.: Judging the spatial relevance of documents for GIR. In: ECIR\u201906: Proceedings of the 28th European Conference on IR Research, LNCS, vol. 3936, pp. 548\u2013552. Springer, 2006. doi: https:\/\/doi.org\/10.1007\/11735106_62","DOI":"10.1007\/11735106_62"},{"key":"70_CR40","doi-asserted-by":"publisher","unstructured":"Martins, B., Silva, M.J., Andrade, L.: Indexing and ranking in Geo-IR systems. In: GIR\u201905: Proceedings of the 2005 Workshop on Geographic Information Retrieval, pp. 31\u201334, New York, NY, USA, 2005. ACM. doi: https:\/\/doi.org\/10.1145\/1096985.1096993","DOI":"10.1145\/1096985.1096993"},{"key":"70_CR41","doi-asserted-by":"publisher","unstructured":"Jones, C.B., Purves, R.: GIR\u201905 2005 ACM workshop on geographical information retrieval. SIGIR Forum, 40(1), 34\u201337 (2006). ISSN 0163-5840. doi: https:\/\/doi.org\/10.1145\/1147197.1147202","DOI":"10.1145\/1147197.1147202"},{"key":"70_CR42","doi-asserted-by":"publisher","unstructured":"Larson, R.R.: Geographic information retrieval and digital libraries. In: ECDL\u201909: Proceedings of the 13th European Conference on Digital Libraries, LNCS, vol. 5714, pp. 461\u2013464. Springer, 2009. doi: https:\/\/doi.org\/10.1007\/978-3-642-04346-8_59","DOI":"10.1007\/978-3-642-04346-8_59"},{"key":"70_CR43","unstructured":"Widl\u00f6cher, A., Bilhaut, F.: La plate-forme LinguaStream: un outil d\u2019exploration linguistique sur corpus. In: TALN\u201905: Actes de la 12e Conf\u00e9rence sur le Traitement Automatique du Langage Naturel (2005)"},{"key":"70_CR44","unstructured":"Kergosien, E., Kamel, M., Sallaberry, C., Bessagnet, M.-N., Aussenac-Gilles, N., Gaio, M.: Construction automatique d\u2019ontologie et enrichissement \u00e0 partir de ressources externes. In: JFO\u201909: Actes des 3e Journ\u00e9es Francophones sur les Ontologies, pp. 11\u201320, 2009"},{"key":"70_CR45","doi-asserted-by":"publisher","unstructured":"Sallaberry, C., Gaio, M., Palacio, D., Lesbegueries, J.: Fuzzying GIS topological functions for GIR needs. In: GIR\u201908: Proceeding of the 2nd international Workshop on Geographic Information Retrieval, pp. 1\u20138, New York, NY, USA, 2008. ACM. ISBN 978-1-60558-253-5. doi: https:\/\/doi.org\/10.1145\/1460007.1460008","DOI":"10.1145\/1460007.1460008"},{"issue":"2","key":"70_CR46","doi-asserted-by":"publisher","first-page":"157","DOI":"10.1080\/02693799008941537","volume":"4","author":"N.R. Chrisman","year":"1990","unstructured":"Chrisman N.R.: Deficiencies of sheets and tiles: building sheetless databases. Int. J. Geogr. Inf. Sci. 4(2), 157\u2013167 (1990). doi: https:\/\/doi.org\/10.1080\/02693799008941537","journal-title":"Int. J. Geogr. Inf. Sci."},{"key":"70_CR47","unstructured":"Palacio, D., Sallaberry, C., Gaio, M.: Normalizing spatial information to improve geographical information indexing and retrieval in digital libraries. In: ISGIS\u201910: Proceedings of the Joint International Conference on Theory, Data Handling and Modelling in GeoSpatial Information Science proceedings, pp. 229\u2013234 (2010)"},{"key":"70_CR48","unstructured":"Palacio, D., Sallaberry, C., Gaio, M.: Normalizing spatial information to better combine criteria in geographical information retrieval. In: ECIR-GIIW\u201909: Proceeding of the International Workshop on Geographic Information on the Internet, pp. 37\u201348, 2009"},{"key":"70_CR49","unstructured":"Fox, E.A., Shaw, J.A.: Combination of multiple searches. In: Harman, D.K. (ed.) TREC-1: Proceedings of the First Text REtrieval Conference, pp. 243\u2013252, Gaithersburg, MD, USA, February 1993. NIST"},{"issue":"8","key":"70_CR50","doi-asserted-by":"publisher","first-page":"1625","DOI":"10.1002\/asi.21091","volume":"60","author":"G. Hubert","year":"2009","unstructured":"Hubert G., Mothe J.: An adaptable search engine for multimodal information retrieval. J. Am. Soc. Inf. Sci. Technol., 60(8), 1625\u20131634 (2009). doi: https:\/\/doi.org\/10.1002\/asi.21091","journal-title":"J. Am. Soc. Inf. Sci. Technol."},{"key":"70_CR51","doi-asserted-by":"publisher","unstructured":"Lee, J.H.: Analyses of multiple evidence combination. In: SIGIR\u201997: Proceedings of the 20th Annual International ACM SIGIR Conference, pp. 267\u2013276, New York, NY, USA, 1997. ACM Press. doi: https:\/\/doi.org\/10.1145\/258525.258587","DOI":"10.1145\/258525.258587"},{"key":"70_CR52","unstructured":"de Borda, J.-C.: M\u00e9moire sur les \u00e9lections au scrutin. In: Histoire de l\u2019Acad\u00e9mie Royale des Sciences, Paris (1781)"},{"key":"70_CR53","unstructured":"de Condorcet, M.: Essai sur l\u2019application de l\u2019analyse \u00e0 la probabilit\u00e9 des d\u00e9cisions rendues \u00e0 la pluralit\u00e9 des voix. Imprimerie Royale, Paris (1785)"},{"key":"70_CR54","doi-asserted-by":"publisher","unstructured":"Saari, D.: Which is better: the Condorcet or Borda winner? Soc. Choice Welfare 26(1), 107\u2013129, 2006. ISSN 0176-1714. doi: https:\/\/doi.org\/10.1007\/s00355-005-0046-2","DOI":"10.1007\/s00355-005-0046-2"},{"key":"70_CR55","doi-asserted-by":"publisher","unstructured":"Aslam, J.A., Montague, M.: Models for metasearch. In: SIGIR\u201901: Proceedings of the 24th Annual International ACM SIGIR Conference, pp. 276\u2013284, New York, NY, USA, 2001. ACM. doi: https:\/\/doi.org\/10.1145\/383952.384007","DOI":"10.1145\/383952.384007"},{"key":"70_CR56","doi-asserted-by":"publisher","unstructured":"Montague, M., Aslam, J.A.: Condorcet fusion for improved retrieval. In: CIKM\u201902: Proceedings of the 11th International Conference on Information and Knowledge Management, pp. 538\u2013548, New York, NY, USA, 2002. ACM. doi: https:\/\/doi.org\/10.1145\/584792.584881","DOI":"10.1145\/584792.584881"},{"key":"70_CR57","doi-asserted-by":"publisher","unstructured":"Cabanac, G., Hubert, G., Boughanem, M., Chrisment, C.: Tie-breaking Bias: Effect of an Uncontrolled Parameter on Information Retrieval Evaluation. In: Agosti, M., Ferro, N., Peters, C., de Rijke, M., Smeaton, A.F. (eds.) CLEF\u201910: Proceedings of the 1st Conference on Multilingual and Multimodal Information Access Evaluation, LNCS, vol. 6360, pp. 112\u2013123. Springer, 2010. doi: https:\/\/doi.org\/10.1007\/978-3-642-15998-5_13","DOI":"10.1007\/978-3-642-15998-5_13"},{"key":"70_CR58","doi-asserted-by":"publisher","unstructured":"Farah, M., Vanderpooten, D.: An outranking approach for rank aggregation in information retrieval. In: SIGIR\u201907: Proceedings of the 30th Annual International ACM SIGIR Conference, pp. 591\u2013598, New York, NY, USA, 2007. ACM. doi: https:\/\/doi.org\/10.1145\/1277741.1277843","DOI":"10.1145\/1277741.1277843"},{"key":"70_CR59","doi-asserted-by":"publisher","unstructured":"Liu, B.: Information Retrieval and Web Search. In: Web Data Mining, Data-Centric Systems and Applications, chapter 6, pp. 183\u2013236. Springer, 2007. ISBN 978-3-540-37882-2. doi: https:\/\/doi.org\/10.1007\/978-3-540-37882-2_6","DOI":"10.1007\/978-3-540-37882-2_6"},{"key":"70_CR60","unstructured":"Cleverdon, C.W.: Report on the testing and analysis of an investigation into the comparative efficiency of indexing systems. ASLIB Cranfield Research Project, ASLIB, Cranfield, UK, October 1962"},{"issue":"4","key":"70_CR61","doi-asserted-by":"publisher","first-page":"439","DOI":"10.1177\/0165551507086989","volume":"34","author":"S. Robertson","year":"2008","unstructured":"Robertson, S.: On the history of evaluation in IR. J. Inf. Sci. 34(4), 439\u2013456, 2008. doi: https:\/\/doi.org\/10.1177\/0165551507086989","journal-title":"J. Inf. Sci."},{"key":"70_CR62","doi-asserted-by":"publisher","unstructured":"Sanderson, M.: Test collection based evaluation of information retrieval systems. Found. Trends Inf. Retr. 4(4):247\u2013375, 2010. ISSN 1554-0669. doi: https:\/\/doi.org\/10.1561\/1500000009","DOI":"10.1561\/1500000009"},{"key":"70_CR63","unstructured":"Harman, D.K.: The TREC test collections. In: Voorhees and Harman [67], chapter 2, pp. 21\u201353"},{"key":"70_CR64","doi-asserted-by":"publisher","unstructured":"Buckley, C., Voorhees, E.M.: Evaluating evaluation measure stability. In: SIGIR\u201900: Proceedings of the 23rd International ACM SIGIR Conference, pp. 33\u201340, New York, NY, USA, 2000. ACM. doi: https:\/\/doi.org\/10.1145\/345508.345543","DOI":"10.1145\/345508.345543"},{"key":"70_CR65","doi-asserted-by":"publisher","unstructured":"Voorhees, E.M.: The philosophy of information retrieval evaluation. In: Peters, C., Braschler, M., Gonzalo, J., Kluck, M. (eds.) CLEF\u201901: Proceedings of the Second Workshop of the Cross-Language Evaluation Forum, LNCS, vol. 2406, pp. 355\u2013370. Springer, 2002. doi: https:\/\/doi.org\/10.1007\/3-540-45691-0_34","DOI":"10.1007\/3-540-45691-0_34"},{"key":"70_CR66","doi-asserted-by":"crossref","unstructured":"Harman, D.K. (ed.): TREC-1: Proceedings of the First Text REtrieval Conference, Gaithersburg, MD, USA, February 1993. NIST","DOI":"10.6028\/NIST.SP.500-207"},{"key":"70_CR67","unstructured":"Voorhees, E.M., Harman, D.K.: TREC: Experiment and Evaluation in Information Retrieval. MIT Press, Cambridge, MA, USA, 2005"},{"key":"70_CR68","unstructured":"Kando, N., Kuriyama, K., Nozue, T., Eguchi, K., Kato, H., Hidaka, S.: Overview of IR Tasks at the First NTCIR Workshop. In: Proceedings of the First NTCIR Workshop on Research in Japanese Text Retrieval and Term Recognition, pp. 11\u201344. NACSIS, 1999"},{"key":"70_CR69","doi-asserted-by":"publisher","unstructured":"Kando, N.: Evaluation of information access technologies at the NTCIR workshop. In: Peters, C., Gonzalo, J., Braschler, M., Kluck, M. (eds.) Comparative Evaluation of Multilingual Information Access Systems, LNCS, vol. 3237, pp. 197\u2013221. Springer, 2004. doi: https:\/\/doi.org\/10.1007\/978-3-540-30222-3_4","DOI":"10.1007\/978-3-540-30222-3_4"},{"key":"70_CR70","doi-asserted-by":"publisher","unstructured":"Peters, C. (ed.): CLEF\u201901: Proceedings of the 2nd Workshop of the Cross-Language Evaluation Forum, LNCS, vol. 2069. Springer, 2001. doi: https:\/\/doi.org\/10.1007\/3-540-44645-1","DOI":"10.1007\/3-540-44645-1"},{"issue":"12","key":"70_CR71","doi-asserted-by":"publisher","first-page":"1067","DOI":"10.1002\/asi.1164","volume":"52","author":"C. Peters","year":"2001","unstructured":"Peters C., Braschler M.: European Research Letter\u2014Cross- Language System Evaluation: the CLEF Campaigns. J. Am. Soc. Inf. Sci. Technol. 52(12), 1067\u20131072 (2001). doi: https:\/\/doi.org\/10.1002\/asi.1164","journal-title":"J. Am. Soc. Inf. Sci. Technol."},{"key":"70_CR72","unstructured":"Fuhr, N., G\u00f6vert, N., Kazai, G., Lalmas, M. (eds.): INEX\u201902: Proceedings of the First Workshop of the INitiative for the Evaluation of XML Retrieval (INEX), 2002"},{"issue":"6","key":"70_CR73","doi-asserted-by":"publisher","first-page":"551","DOI":"10.1002\/asi.10386","volume":"55","author":"G. Kazai","year":"2004","unstructured":"Kazai G., Lalmas M., Fuhr N., Fuhr N., Fuhr N.: A report on the first year of the initiative for the evaluation of XML retrieval. J. Am. Soc. Inf. Sci. Technol. 55(6), 551\u2013556 (2004). doi: https:\/\/doi.org\/10.1002\/asi.10386","journal-title":"J. Am. Soc. Inf. Sci. Technol."},{"issue":"2","key":"70_CR74","doi-asserted-by":"publisher","first-page":"161","DOI":"10.1007\/s10579-009-9086-z","volume":"43","author":"M. Verhagen","year":"2009","unstructured":"Verhagen M., Gaizauskas R., Schilder F., Hepple M., Moszkowicz J., Pustejovsky J.: The TempEval challenge: identifying temporal relations in text. Lang. Resour. Eval. 43(2), 161\u2013179 (2009). doi: https:\/\/doi.org\/10.1007\/s10579-009-9086-z","journal-title":"Lang. Resour. Eval."},{"key":"70_CR75","unstructured":"Bucher, B., Clough, P., Joho, H., Purves, R., Syed, A.K.: Geographic IR systems: Requirements and evaluation. In: ICC\u201905: Proceedings of the 22nd International Cartographic Conference. Global Congressos, 2005. CDROM"},{"key":"70_CR76","doi-asserted-by":"publisher","unstructured":"Gey, F.C., Larson, R.R., Sanderson, M., Joho, H., Clough, P., Petras, V.: GeoCLEF\u201905: the CLEF 2005 cross-language geographic information retrieval track overview. In: CLEF\u201905: Proceedings of the 5th Workshop on Cross-Language Evalution Forum, LNCS, vol. 4022, pp. 908\u2013919. Springer, 2006. doi: https:\/\/doi.org\/10.1007\/11878773_101","DOI":"10.1007\/11878773_101"},{"key":"70_CR77","doi-asserted-by":"publisher","unstructured":"Perea-Ortega, J.M., Garc\u00eda-Cumbreras, M.A., Garc\u00eda-Vega, M., Ure\u00f1a-L\u00f3pez, L. A. Comparing several textual information retrieval systems for the geographical information retrieval task. In: NLDB\u201908: Proceedings of the 13th International Conference on Natural Language and Information Systems, pp. 142\u2013147, Berlin, Heidelberg, 2008. Springer-Verlag. doi: https:\/\/doi.org\/10.1007\/978-3-540-69858-6_15","DOI":"10.1007\/978-3-540-69858-6_15"},{"key":"70_CR78","doi-asserted-by":"crossref","unstructured":"Ogilvie, P., Callan, J.P.: Experiments using the Lemur toolkit. In: TREC\u201901: Proceedings of the 9th Text REtrieval Conference, Gaithersburg, MD, USA, February 2001. NIST","DOI":"10.6028\/NIST.SP.500-250.web-cmu-lti"},{"key":"70_CR79","unstructured":"Gospodneti\u0107, O., Hatcher, E.: Lucene in Action. Manning Publications (2005)"},{"key":"70_CR80","unstructured":"Gey, F., Larson, R., Kando, N., Machado, J., Sakai, T.: NTCIR-GeoTime Overview: Evaluating Geographic and Temporal Search. In: NTCIR\u201910: Proceedings of the 8th NTCIR Workshop, pp. 147\u2013153, Tockyo, Japan, 2010. NII"},{"key":"70_CR81","doi-asserted-by":"publisher","unstructured":"Santos, D., Cabral, L.: GikiCLEF: Expectations and Lessons Learned. In: Peters, C., Di Nunzio, G., Kurimo, M., Mostefa, D., Penas, A., Roda, G. (eds.) CLEF\u201909: Proceedings of the 9th Workshop of the Cross-Language Evaluation Forum, LNCS, vol. 6241, pp. 212\u2013222. Springer Berlin\/Heidelberg, 2010. doi: https:\/\/doi.org\/10.1007\/978-3-642-15754-7_23","DOI":"10.1007\/978-3-642-15754-7_23"},{"key":"70_CR82","doi-asserted-by":"publisher","unstructured":"Palacio, D., Cabanac, G., Sallaberry, C., Hubert, G.: Measuring effectiveness of geographic IR systems in digital libraries: evaluation framework and case study. In: ECDL\u201910: Proceedings of the 14th European Conference on Research and Advanced Technology for Digital Libraries, LNCS, vol. 6273, pp. 340\u2013351. Springer, 2010. doi: https:\/\/doi.org\/10.1007\/978-3-642-15464-5_34","DOI":"10.1007\/978-3-642-15464-5_34"},{"issue":"4","key":"70_CR83","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K. J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin K., Kek\u00e4l\u00e4inen J.: Cumulated gain-based evaluation of IR techniques. ACM Trans. Inf. Syst. 20(4), 422\u2013446 (2002). doi: https:\/\/doi.org\/10.1145\/582415.582418","journal-title":"ACM Trans. Inf. Syst."},{"key":"70_CR84","doi-asserted-by":"publisher","unstructured":"Hull, D.: Using statistical testing in the evaluation of retrieval experiments. In: SIGIR\u201993: Proceedings of the 16th Annual International ACM SIGIR Conference, pp. 329\u2013338, New York, NY, USA, 1993. ACM Press. doi: https:\/\/doi.org\/10.1145\/160688.160758","DOI":"10.1145\/160688.160758"},{"key":"70_CR85","doi-asserted-by":"publisher","unstructured":"da Costa Pereira, C., Dragoni, M., Pasi, G.: Multidimensional relevance: A new aggregation criterion. In: ECIR\u201909: Proceedings of the 31th European Conference on IR Research on Advances in Information Retrieval, pp. 264\u2013275, Berlin, Heidelberg (2009). Springer-Verlag. doi: https:\/\/doi.org\/10.1007\/978-3-642-00958-7_25","DOI":"10.1007\/978-3-642-00958-7_25"},{"issue":"4","key":"70_CR86","doi-asserted-by":"publisher","first-page":"315","DOI":"10.1007\/s10791-008-9046-z","volume":"11","author":"M. Farah","year":"2008","unstructured":"Farah M., Vanderpooten D.: An outranking approach for information retrieval. Inf. Retr. 11(4), 315\u2013334 (2008). doi: https:\/\/doi.org\/10.1007\/s10791-008-9046-z","journal-title":"Inf. Retr."},{"key":"70_CR87","doi-asserted-by":"publisher","unstructured":"Andogah, G., Bouma, G.: Relevance measures using geographic scopes and types. In: CLEF\u201907: Proceedings of the 7th Workshop of the Cross-Language Evaluation Forum, LNCS, vol. 5152, pp. 794\u2013801 (2008). doi: https:\/\/doi.org\/10.1007\/978-3-540-85760-0_100","DOI":"10.1007\/978-3-540-85760-0_100"},{"issue":"1","key":"70_CR88","doi-asserted-by":"publisher","first-page":"183","DOI":"10.1109\/21.87068","volume":"18","author":"R.R. Yager","year":"1988","unstructured":"Yager R.R.: On ordered weighted averaging aggregation operators in multicriteria decisionmaking. IEEE Trans. Syst. Man Cybern. 18(1), 183\u2013190 (1988). doi: https:\/\/doi.org\/10.1109\/21.87068","journal-title":"IEEE Trans. Syst. Man Cybern."},{"key":"70_CR89","doi-asserted-by":"publisher","unstructured":"Boughanem, M., Loiseau, Y., Prade, H.: Refining aggregation functions for improving document ranking in information retrieval. In: SUM\u201907: Proceedings of the 1st International Conference on Scalable Uncertainty Management, pp. 255\u2013267, Berlin, Heidelberg, 2007. Springer. doi: https:\/\/doi.org\/10.1007\/978-3-540-75410-7_19","DOI":"10.1007\/978-3-540-75410-7_19"},{"issue":"1","key":"70_CR90","doi-asserted-by":"publisher","first-page":"11","DOI":"10.1016\/S0165-0114(02)00429-3","volume":"137","author":"C. Labreuche","year":"2003","unstructured":"Labreuche C., Grabisch M.: The Choquet integral for the aggregation of interval scales in multicriteria decision making. Fuzzy Sets Syst. 137(1), 11\u201326 (2003). doi: https:\/\/doi.org\/10.1016\/S0165-0114(02)00429-3","journal-title":"Fuzzy Sets Syst."}],"container-title":["International Journal on Digital Libraries"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s00799-011-0070-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s00799-011-0070-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s00799-011-0070-z","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s00799-011-0070-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,3,12]],"date-time":"2025-03-12T03:00:33Z","timestamp":1741748433000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s00799-011-0070-z"}},"subtitle":["Evaluation framework and case study"],"short-title":[],"issued":{"date-parts":[[2010,6]]},"references-count":90,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2010,6]]}},"alternative-id":["70"],"URL":"https:\/\/doi.org\/10.1007\/s00799-011-0070-z","relation":{},"ISSN":["1432-5012","1432-1300"],"issn-type":[{"value":"1432-5012","type":"print"},{"value":"1432-1300","type":"electronic"}],"subject":[],"published":{"date-parts":[[2010,6]]}}}