{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T00:58:01Z","timestamp":1740099481813,"version":"3.37.3"},"publisher-location":"Cham","reference-count":32,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030229474"},{"type":"electronic","value":"9783030229481"}],"license":[{"start":{"date-parts":[[2019,1,1]],"date-time":"2019-01-01T00:00:00Z","timestamp":1546300800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2019]]},"DOI":"10.1007\/978-3-030-22948-1_9","type":"book-chapter","created":{"date-parts":[[2019,8,13]],"date-time":"2019-08-13T05:02:59Z","timestamp":1565672579000},"page":"217-230","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Multi-Lingual Retrieval of Pictures in ImageCLEF"],"prefix":"10.1007","author":[{"given":"Paul","family":"Clough","sequence":"first","affiliation":[]},{"given":"Theodora","family":"Tsikrika","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,8,14]]},"reference":[{"key":"9_CR1","doi-asserted-by":"publisher","first-page":"5","DOI":"10.1145\/1498759.1498766","volume-title":"Proceedings of the second ACM international conference on web search and data mining, WSDM\u201809","author":"R Agrawal","year":"2009","unstructured":"Agrawal R, Gollapudi S, Halverson A, Ieong S (2009) Diversifying search results. In: Proceedings of the second ACM international conference on web search and data mining, WSDM\u201809. ACM, New York, pp 5\u201314. http:\/\/doi.acm.org\/10.1145\/1498759.1498766"},{"key":"9_CR2","series-title":"Revised Selected Papers, Lecture notes in computer science (LNCS), vol","doi-asserted-by":"publisher","first-page":"500","DOI":"10.1007\/978-3-642-04447-2_62","volume-title":"Evaluating systems for multilingual and multimodal information access: ninth workshop of the cross\u2013language evaluation forum (CLEF 2008)","author":"T Arni","year":"2009","unstructured":"Arni T, Clough P, Sanderson M, Grubinger M (2009) Overview of the ImageCLEFphoto 2008 photographic retrieval task. In: Peters C, Deselaers T, Ferro N, Gonzalo J, Jones GJF, Kurimo M, Mandl T, Pe\u00f1as A (eds) Evaluating systems for multilingual and multimodal information access: ninth workshop of the cross\u2013language evaluation forum (CLEF 2008). Revised Selected Papers, Lecture notes in computer science (LNCS), vol 5706. Springer, Heidelberg, pp 500\u2013511"},{"key":"9_CR3","doi-asserted-by":"publisher","first-page":"743","DOI":"10.1145\/2567948.2579323","volume-title":"Proceedings of the 23rd international conference on world wide web, WWW\u201814 companion","author":"C Boididou","year":"2014","unstructured":"Boididou C, Papadopoulos S, Kompatsiaris Y, Schifferes S, Newman N (2014) Challenges of computational verification in social multimedia. In: Proceedings of the 23rd international conference on world wide web, WWW\u201814 companion. ACM, New York, pp 743\u2013748. http:\/\/doi.acm.org\/10.1145\/2567948.2579323"},{"key":"9_CR4","doi-asserted-by":"publisher","first-page":"33","DOI":"10.1145\/345508.345543","volume-title":"Proceedings of the 23rd annual international ACM SIGIR conference on research and development in information retrieval, SIGIR\u201800","author":"C Buckley","year":"2000","unstructured":"Buckley C, Voorhees EM (2000) Evaluating evaluation measure stability. In: Proceedings of the 23rd annual international ACM SIGIR conference on research and development in information retrieval, SIGIR\u201800. ACM, New York, pp 33\u201340. http:\/\/doi.acm.org\/10.1145\/345508.345543"},{"key":"9_CR5","first-page":"25","volume-title":"Proceedings of the 27th annual international ACM SIGIR conference on research and development in information retrieval, SIGIR\u201804","author":"C Buckley","year":"2004","unstructured":"Buckley C, Voorhees EM (2004) Retrieval evaluation with incomplete information. In: Proceedings of the 27th annual international ACM SIGIR conference on research and development in information retrieval, SIGIR\u201804. ACM, New York, pp 25\u201332. http:\/\/doi.acm.org\/10.1145\/1008992.1009000"},{"key":"9_CR6","doi-asserted-by":"publisher","first-page":"659","DOI":"10.1145\/1390334.1390446","volume-title":"Proceedings of the 31st annual international ACM SIGIR conference on research and development in information retrieval, SIGIR\u201808","author":"CL Clarke","year":"2008","unstructured":"Clarke CL, Kolla M, Cormack GV, Vechtomova O, Ashkan A, B\u00fcttcher S, MacKinnon I (2008) Novelty and diversity in information retrieval evaluation. In: Proceedings of the 31st annual international ACM SIGIR conference on research and development in information retrieval, SIGIR\u201808. ACM, New York, pp 659\u2013666. http:\/\/doi.acm.org\/10.1145\/1390334.1390446"},{"key":"9_CR7","series-title":"Lecture notes in computer science (LNCS), vol","doi-asserted-by":"publisher","first-page":"581","DOI":"10.1007\/978-3-540-30222-3_56","volume-title":"Comparative evaluation of multilingual information access systems: fourth workshop of the cross\u2013language evaluation forum (CLEF 2003), revised selected papers","author":"P Clough","year":"2004","unstructured":"Clough P, Sanderson M (2004) The CLEF 2003 cross language image retrieval track. In: Peters C, Braschler M, Gonzalo J, Kluck M (eds) Comparative evaluation of multilingual information access systems: fourth workshop of the cross\u2013language evaluation forum (CLEF 2003), revised selected papers. Lecture notes in computer science (LNCS), vol 3237. Springer, Heidelberg, pp 581\u2013593"},{"key":"9_CR8","series-title":"Lecture notes in computer science (LNCS), vol","doi-asserted-by":"publisher","first-page":"597","DOI":"10.1007\/11519645_59","volume-title":"Multilingual information access for text, speech and images: fifth workshop of the cross\u2013language evaluation forum (CLEF 2004). Revised selected papers","author":"P Clough","year":"2005","unstructured":"Clough P, M\u00fcller H, Sanderson M (2005) The CLEF 2004 cross-language image retrieval track. In: Peters C, Clough P, Gonzalo J, Jones GJF, Kluck M, Magnini B (eds) Multilingual information access for text, speech and images: fifth workshop of the cross\u2013language evaluation forum (CLEF 2004). Revised selected papers. Lecture notes in computer science (LNCS), vol 3491. Springer, Heidelberg, pp 597\u2013613"},{"doi-asserted-by":"crossref","unstructured":"Clough P, Grubinger M, Deselaers T, Hanbury A, M\u00fcller H (2006a) Overview of the ImageCLEF 2006 photographic retrieval and object annotation tasks. In: Nardi A, Peters C, Vicedo JL, Ferro N (eds) CLEF 2006 working notes. CEUR workshop proceedings (CEUR-WS.org). ISSN 1613-0073. http:\/\/ceur-ws.org\/Vol-1172\/","key":"9_CR9","DOI":"10.1007\/978-3-540-74999-8_71"},{"issue":"1","key":"9_CR10","doi-asserted-by":"publisher","first-page":"21","DOI":"10.1145\/1147197.1147199","volume":"40","author":"PD Clough","year":"2006","unstructured":"Clough PD, Sanderson M, Reid N (2006b) The eurovision St Andrews collection of photographs. SIGIR Forum 40(1):21\u201330. http:\/\/doi.acm.org\/10.1145\/1147197.1147199","journal-title":"SIGIR Forum"},{"key":"9_CR11","series-title":"Revised selected papers, Lecture notes in computer science (LNCS), vol","doi-asserted-by":"publisher","first-page":"223","DOI":"10.1007\/978-3-540-74999-8_31","volume-title":"Evaluation of multilingual and multi-modal information retrieval: seventh workshop of the cross\u2013language evaluation forum (CLEF 2006)","author":"P Clough","year":"2007","unstructured":"Clough P, Grubinger M, Deselaers T, Hanbury A, M\u00fcller H (2007) Overview of the ImageCLEF 2006 photographic retrieval and object annotation tasks. In: Peters C, Clough P, Gey FC, Karlgren J, Magnini B, Oard DW, de Rijke M, Stempfhuber M (eds) Evaluation of multilingual and multi-modal information retrieval: seventh workshop of the cross\u2013language evaluation forum (CLEF 2006). Revised selected papers, Lecture notes in computer science (LNCS), vol 4730. Springer, Heidelberg, pp 223\u2013256"},{"key":"9_CR12","volume-title":"Visual information retrieval","author":"A Bimbo Del","year":"1999","unstructured":"Del Bimbo A (1999) Visual information retrieval. Morgan Kaufmann, San Francisco"},{"issue":"2","key":"9_CR13","first-page":"10","volume":"28","author":"M Grubinger","year":"2006","unstructured":"Grubinger M, Clough P, Leung C (2006) The IAPR TC-12 benchmark for visual information search. IAPR Newsl 28(2):10\u201312","journal-title":"IAPR Newsl"},{"key":"9_CR14","series-title":"Revised selected papers, Lecture notes in computer science (LNCS), vol","doi-asserted-by":"publisher","first-page":"433","DOI":"10.1007\/978-3-540-85760-0_57","volume-title":"Advances in multilingual and multimodal information retrieval: eighth workshop of the cross\u2013language evaluation forum (CLEF 2007)","author":"M Grubinger","year":"2008","unstructured":"Grubinger M, Clough P, Hanbury A, M\u00fcller H (2008) Overview of the ImageCLEFphoto 2007 photographic retrieval task. In: Peters C, Jijkoun V, Mandl T, M\u00fcller H, Oard DW, Pe\u00f1as A, Petras V, Santos D (eds) Advances in multilingual and multimodal information retrieval: eighth workshop of the cross\u2013language evaluation forum (CLEF 2007). Revised selected papers, Lecture notes in computer science (LNCS), vol 5152. Springer, Heidelberg, pp 433\u2013444"},{"issue":"9","key":"9_CR15","doi-asserted-by":"publisher","first-page":"1783","DOI":"10.1109\/TPAMI.2016.2613873","volume":"39","author":"SS Husain","year":"2017","unstructured":"Husain SS, Bober M (2017) Improving large-scale image retrieval through robust aggregation of local descriptors. IEEE Trans Pattern Anal Mach Intell 39(9):1783\u20131796. https:\/\/doi.org\/10.1109\/TPAMI.2016.2613873","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"unstructured":"M\u00fcller H, Deselaers T, Grubinger M, Clough P, Hanbury A, Hersh W (2007) Problems with running a successful multimedia retrieval benchmark. In: Proceedings of the third MUSCLE\/ImageCLEF workshop on image and video retrieval evaluation","key":"9_CR16"},{"key":"9_CR17","first-page":"72","volume-title":"Revised selected papers, Lecture notes in computer science (LNCS)","author":"M\u00fcller H, Kalpathy-Cramer J, Eggel I, Bedrick S, Radhouani S, Bakke B, Khan CE, Jr, Hersh WR (2010) Overview of the CLEF 2009 medical image retrieval track. In: Peters C, Tsikrika T, M\u00fcller H, Kalpathy-Cramer J, Jones GJF, Gonzalo J, Caputo B (eds) Multilingual information access evaluation vol. ii multimedia experiments \u2013 tenth workshop of the cross\u2013language evaluation forum (CLEF","year":"2009","unstructured":"M\u00fcller H, Kalpathy-Cramer J, Eggel I, Bedrick S, Radhouani S, Bakke B, Khan CE, Jr, Hersh WR (2010) Overview of the CLEF 2009 medical image retrieval track. In: Peters C, Tsikrika T, M\u00fcller H, Kalpathy-Cramer J, Jones GJF, Gonzalo J, Caputo B (eds) Multilingual information access evaluation vol. ii multimedia experiments \u2013 tenth workshop of the cross\u2013language evaluation forum (CLEF 2009). Revised selected papers, Lecture notes in computer science (LNCS). Springer, Heidelberg, pp 72\u201384"},{"key":"9_CR18","doi-asserted-by":"publisher","first-page":"141","DOI":"10.1007\/978-3-642-15181-1_8","volume-title":"ImageCLEF, experimental evaluation in visual information retrieval","author":"ML Paramita","year":"2010","unstructured":"Paramita ML, Grubinger M (2010) Photographic image retrieval. In: M\u00fcller H, Clough PD, Deselaers T, Caputo B (eds) ImageCLEF, experimental evaluation in visual information retrieval. Springer, Berlin, pp 141\u2013162. https:\/\/doi.org\/10.1007\/978-3-642-15181-1_8"},{"key":"9_CR19","first-page":"39","volume":"9","author":"ML Paramita","year":"2009","unstructured":"Paramita ML, Sanderson M, Clough P (2009a) Developing a test collection to support diversity analysis. In: Proceedings of redundancy, diversity, and IDR workshop-SIGIR, vol 9, pp 39\u201345","journal-title":"Proceedings of redundancy, diversity, and IDR workshop-SIGIR, vol"},{"doi-asserted-by":"crossref","unstructured":"Paramita ML, Sanderson M, Clough P (2009b) Diversity in photo retrieval: overview of the imageclefphoto task 2009. In: Workshop of the cross-language evaluation forum for European languages. Springer, Berlin pp 45\u201359","key":"9_CR20","DOI":"10.1007\/978-3-642-15751-6_6"},{"unstructured":"Popescu A, Tsikrika T, Kludas J (2010) Overview of the wikipedia retrieval task at ImageCLEF 2010. In: Braschler M, Harman DK, Pianta E, Ferro N (eds) CLEF 2010 working notes, CEUR workshop proceedings (CEUR-WS.org). ISSN 1613-0073. http:\/\/ceur-ws.org\/Vol-1176\/","key":"9_CR21"},{"issue":"3","key":"9_CR22","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1007\/s11263-015-0816-y","volume":"115","author":"O Russakovsky","year":"2015","unstructured":"Russakovsky O, Deng J, Su H, Krause J, Satheesh S, Ma S, Huang Z, Karpathy A, Khosla A, Bernstein M, Berg AC, Fei-Fei L (2015) Imagenet large scale visual recognition challenge. Int J Comput Vis 115(3):211\u2013252. https:\/\/doi.org\/10.1007\/s11263-015-0816-y","journal-title":"Int J Comput Vis"},{"issue":"1","key":"9_CR23","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1561\/1500000040","volume":"9","author":"RLT Santos","year":"2015","unstructured":"Santos RLT, Macdonald C, Ounis I (2015) Search result diversification. Found Trends Inf Retr 9(1):1\u201390. http:\/\/dx.doi.org\/10.1561\/1500000040","journal-title":"Found Trends Inf Retr"},{"key":"9_CR24","series-title":"Revised selected papers, Lecture notes in computer science (LNCS), vol","doi-asserted-by":"publisher","first-page":"539","DOI":"10.1007\/978-3-642-04447-2_66","volume-title":"Evaluating systems for multilingual and multimodal information access: ninth workshop of the cross\u2013language evaluation forum (CLEF 2008)","author":"T Tsikrika","year":"2009","unstructured":"Tsikrika T, Kludas J (2009) Overview of the WikipediaMM task at ImageCLEF 2008. In: Peters C, Deselaers T, Ferro N, Gonzalo J, Jones GJF, Kurimo M, Mandl T, Pe\u00f1as A (eds) Evaluating systems for multilingual and multimodal information access: ninth workshop of the cross\u2013language evaluation forum (CLEF 2008). Revised selected papers, Lecture notes in computer science (LNCS), vol 5706. Springer, Heidelberg, pp 539\u2013550"},{"key":"9_CR25","first-page":"60","volume-title":"Revised selected papers, Lecture notes in computer science (LNCS)","author":"Tsikrika T, Kludas J (2010) Overview of the WikipediaMM task at ImageCLEF 2009. In: Peters C, Tsikrika T, M\u00fcller H, Kalpathy-Cramer J, Jones GJF, Gonzalo J, Caputo B (eds) Multilingual information access evaluation vol. ii multimedia experiments \u2013 tenth workshop of the cross\u2013language evaluation forum (CLEF","year":"2009","unstructured":"Tsikrika T, Kludas J (2010) Overview of the WikipediaMM task at ImageCLEF 2009. In: Peters C, Tsikrika T, M\u00fcller H, Kalpathy-Cramer J, Jones GJF, Gonzalo J, Caputo B (eds) Multilingual information access evaluation vol. ii multimedia experiments \u2013 tenth workshop of the cross\u2013language evaluation forum (CLEF 2009). Revised selected papers, Lecture notes in computer science (LNCS). Springer, Heidelberg, pp 60\u201371"},{"key":"9_CR26","doi-asserted-by":"publisher","first-page":"95","DOI":"10.1007\/978-3-642-23708-9_12","volume-title":"Multilingual and Multimodal Information Access Evaluation","author":"Theodora Tsikrika","year":"2011","unstructured":"Tsikrika T, Garcia Seco de Herrera A, M\u00fcller H (2011a) Assessing the scholarly impact of ImageCLEF. In: Forner P, Gonzalo J, Kek\u00e4l\u00e4inen J, Lalmas M, de Rijke M (eds) Multilingual and multimodal information access evaluation. Proceedings of the second international conference of the cross-language evaluation forum (CLEF 2011). Lecture notes in computer science (LNCS), vol 6941. Springer, Heidelberg, pp 95\u2013106"},{"unstructured":"Tsikrika T, Popescu A, Kludas J (2011b) Overview of the Wikipedia image retrieval task at ImageCLEF 2011. In: Petras V, Forner P, Clough P, Ferro N (eds) CLEF 2011 working notes. CEUR workshop proceedings (CEUR-WS.org). ISSN 1613-0073. http:\/\/ceur-ws.org\/Vol-1177\/","key":"9_CR27"},{"issue":"3","key":"9_CR28","doi-asserted-by":"publisher","first-page":"24","DOI":"10.1109\/MMUL.2012.17","volume":"19","author":"T Tsikrika","year":"2012","unstructured":"Tsikrika T, Kludas J, Popescu A (2012) Building reliable and reusable test collections for image retrieval: the wikipedia task at imageclef. IEEE MultiMedia 19(3):24\u201333","journal-title":"IEEE MultiMedia"},{"key":"9_CR29","series-title":"Lecture notes in computer science (LNCS), vol","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/978-3-642-40802-1_1","volume-title":"Information access evaluation meets multilinguality, multimodality, and visualization. Proceedings of the fourth international conference of the CLEF initiative (CLEF 2013)","author":"T Tsikrika","year":"2013","unstructured":"Tsikrika T, Larsen B, M\u00fcller H, Endrullis S, Rahm E (2013) The scholarly impact of CLEF (2000\u20132009). In: Forner P, M\u00fcller H, Paredes R, Rosso P, Stein B (eds) Information access evaluation meets multilinguality, multimodality, and visualization. Proceedings of the fourth international conference of the CLEF initiative (CLEF 2013). Lecture notes in computer science (LNCS), vol 8138. Springer, Heidelberg, pp 1\u201312"},{"key":"9_CR30","doi-asserted-by":"crossref","first-page":"157","DOI":"10.1145\/2647868.2654948","volume-title":"Proceedings of the 22Nd ACM international conference on multimedia, MM\u201814","author":"J Wan","year":"2014","unstructured":"Wan J, Wang D, Hoi SCH, Wu P, Zhu J, Zhang Y, Li J (2014) Deep learning for content-based image retrieval: a comprehensive study. In: Proceedings of the 22Nd ACM international conference on multimedia, MM\u201814. ACM, New York, pp 157\u2013166. http:\/\/doi.acm.org\/10.1145\/2647868.2654948"},{"issue":"1","key":"9_CR31","doi-asserted-by":"publisher","first-page":"58","DOI":"10.1145\/1273221.1273226","volume":"41","author":"T Westerveld","year":"2007","unstructured":"Westerveld T, van Zwol R (2007) Multimedia retrieval at inex 2006. ACM SIGIR Forum 41(1):58\u201363","journal-title":"ACM SIGIR Forum"},{"key":"9_CR32","first-page":"10","volume-title":"Proceedings of the 26th annual international ACM SIGIR conference on research and development in information retrieval, SIGIR\u201803","author":"CX Zhai","year":"2003","unstructured":"Zhai CX, Cohen WW, Lafferty J (2003) Beyond independent relevance: methods and evaluation metrics for subtopic retrieval. In: Proceedings of the 26th annual international ACM SIGIR conference on research and development in information retrieval, SIGIR\u201803. ACM, New York, pp 10\u201317. http:\/\/doi.acm.org\/10.1145\/860435.860440"}],"container-title":["The Information Retrieval Series","Information Retrieval Evaluation in a Changing World"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-22948-1_9","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,1,16]],"date-time":"2021-01-16T14:49:38Z","timestamp":1610808578000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-22948-1_9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019]]},"ISBN":["9783030229474","9783030229481"],"references-count":32,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-22948-1_9","relation":{},"ISSN":["1387-5264"],"issn-type":[{"type":"print","value":"1387-5264"}],"subject":[],"published":{"date-parts":[[2019]]},"assertion":[{"value":"14 August 2019","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}}]}}