{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T01:01:14Z","timestamp":1774400474206,"version":"3.50.1"},"publisher-location":"Cham","reference-count":55,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030229474","type":"print"},{"value":"9783030229481","type":"electronic"}],"license":[{"start":{"date-parts":[[2019,1,1]],"date-time":"2019-01-01T00:00:00Z","timestamp":1546300800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2019]]},"DOI":"10.1007\/978-3-030-22948-1_1","type":"book-chapter","created":{"date-parts":[[2019,8,13]],"date-time":"2019-08-13T01:02:59Z","timestamp":1565658179000},"page":"3-44","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["From Multilingual to Multimodal: The Evolution of CLEF over Two Decades"],"prefix":"10.1007","author":[{"given":"Nicola","family":"Ferro","sequence":"first","affiliation":[]},{"given":"Carol","family":"Peters","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,8,14]]},"reference":[{"key":"1_CR1","unstructured":"Amig\u00f3 E, Corujo A, Gonzalo J, Meij E, de Rijke M (2012) Overview of RepLab 2012: evaluating online reputation management systems. In: Forner P, Karlgren J, Womser-Hacker C, Ferro N (eds) CLEF 2012 working notes, CEUR workshop proceedings (CEUR-WS.org), ISSN 1613-0073. http:\/\/ceur-ws.org\/Vol-1178\/"},{"key":"1_CR2","doi-asserted-by":"crossref","unstructured":"Amig\u00f3 E, Gonzalo J, Verdejo MF (2013) A general evaluation measure for document organization tasks. In: Jones GJF, Sheridan P, Kelly D, de Rijke M, Sakai T (eds) Proc. 36th annual international ACM SIGIR conference on research and development in information retrieval (SIGIR 2013). ACM Press, New York, pp 643\u2013652","DOI":"10.1145\/2484028.2484081"},{"key":"1_CR3","doi-asserted-by":"publisher","first-page":"133","DOI":"10.1016\/j.procs.2014.10.022","volume":"38","author":"Marco Angelini","year":"2014","unstructured":"Angelini M, Ferro N, Larsen B, M\u00fcller H, Santucci G, Silvello G, Tsikrika T (2014) Measuring and analyzing the scholarly impact of experimental evaluation initiatives. In: Agosti M, Catarci T, Esposito F (eds) Proc. 10th Italian research conference on digital libraries (IRCDL 2014). Procedia computer science, vol. 38, pp 133\u2013137","journal-title":"Procedia Computer Science"},{"issue":"6","key":"1_CR4","doi-asserted-by":"publisher","first-page":"1077","DOI":"10.1016\/j.ipm.2018.04.006","volume":"54","author":"M Angelini","year":"2018","unstructured":"Angelini M, Fazzini V, Ferro N, Santucci G, Silvello G (2018) CLAIRE: a combinatorial visual analytics system for information retrieval evaluation. Inf Process Manag 54(6):1077\u20131100","journal-title":"Inf Process Manag"},{"key":"1_CR5","unstructured":"Bollmann P (1984) Two axioms for evaluation measures in information retrieval. In: van Rijsbergen CJ (ed) Proc. of the third joint BCS and ACM symposium on research and development in information retrieval. Cambridge University Press, Cambridge, pp 233\u2013245"},{"issue":"1\/2","key":"1_CR6","doi-asserted-by":"publisher","first-page":"183","DOI":"10.1023\/B:INRT.0000009445.19495.46","volume":"7","author":"M Braschler","year":"2004","unstructured":"Braschler M (2004) Combination approaches for multilingual text retrieval. Inf Retr 7(1\/2):183\u2013204","journal-title":"Inf Retr"},{"key":"1_CR7","doi-asserted-by":"crossref","unstructured":"Buckley C, Voorhees EM (2004) Retrieval evaluation with incomplete information. In: Sanderson M, J\u00e4rvelin K, Allan J, Bruza P (eds) Proc. 27th annual international ACM SIGIR conference on research and development in information retrieval (SIGIR 2004). ACM Press, New York, pp 25\u201332","DOI":"10.1145\/1008992.1009000"},{"key":"1_CR8","first-page":"53","volume-title":"Harman DK","author":"C Buckley","year":"2005","unstructured":"Buckley C, Voorhees EM (2005) Retrieval system evaluation. In: Harman DK, Voorhees EM (eds) TREC. Experiment and evaluation in information retrieval. MIT Press, Cambridge, pp 53\u201378"},{"key":"1_CR9","doi-asserted-by":"crossref","unstructured":"Busin L, Mizzaro S (2013) Axiometrics: an axiomatic approach to information retrieval effectiveness metrics. In: Kurland O, Metzler D, Lioma C, Larsen B, Ingwersen P (eds) Proc. 4th international conference on the theory of information retrieval (ICTIR 2013). ACM Press, New York, pp 22\u201329","DOI":"10.1145\/2499178.2499182"},{"key":"1_CR10","doi-asserted-by":"crossref","unstructured":"B\u00fcttcher S, Clarke CLA, Yeung PCK, Soboroff I (2007) Reliable information retrieval evaluation with incomplete and biased judgements. In: Kraaij W, de Vries AP, Clarke CLA, Fuhr N, Kando N (eds) Proc. 30th annual international ACM SIGIR conference on research and development in information retrieval (SIGIR 2007). ACM Press, New York, pp 63\u201370","DOI":"10.1145\/1277741.1277755"},{"issue":"1","key":"1_CR11","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/2094072.2094076","volume":"30","author":"Benjamin A. Carterette","year":"2012","unstructured":"Carterette BA (2012) Multiple testing in statistical analysis of systems-based information retrieval experiments. ACM Trans Inf Syst 30(1):4:1\u20134:34","journal-title":"ACM Transactions on Information Systems"},{"key":"1_CR12","doi-asserted-by":"crossref","unstructured":"Chapelle O, Metzler D, Zhang Y, Grinspan P (2009) Expected reciprocal rank for graded relevance. In: Cheung DWL, Song IY, Chu WW, Hu X, Lin JJ (eds) Proc. 18th international conference on information and knowledge management (CIKM 2009). ACM Press, New York, pp 621\u2013630","DOI":"10.1145\/1645953.1646033"},{"issue":"6","key":"1_CR13","doi-asserted-by":"publisher","first-page":"173","DOI":"10.1108\/eb050097","volume":"19","author":"CW Cleverdon","year":"1967","unstructured":"Cleverdon CW (1967) The cranfield tests on index languages devices. ASLIB Proc 19(6):173\u2013194","journal-title":"ASLIB Proc"},{"key":"1_CR14","doi-asserted-by":"crossref","unstructured":"Di Nunzio GM, Leveling J, Mandl T (2011) LogCLEF 2011 multilingual log file analysis: language identification, query classification, and success of a query. In: Petras V, Forner P, Clough P, Ferro N (eds) CLEF 2011 working notes. CEUR workshop proceedings (CEUR-WS.org), ISSN 1613-0073. http:\/\/ceur-ws.org\/Vol-1177\/","DOI":"10.1145\/2047403.2047413"},{"key":"1_CR15","doi-asserted-by":"crossref","unstructured":"Ferrante M, Ferro N, Maistro M (2015) Towards a formal framework for utility-oriented measurements of retrieval effectiveness. In: Allan J, Croft WB, de Vries AP, Zhai C, Fuhr N, Zhang Y (eds) Proc. 1st ACM SIGIR international conference on the theory of information retrieval (ICTIR 2015). ACM Press, New York, pp 21\u201330","DOI":"10.1145\/2808194.2809452"},{"key":"1_CR16","doi-asserted-by":"crossref","unstructured":"Ferrante M, Ferro N, Pontarollo S (2017) Are IR evaluation measures on an interval scale? In: Kamps J, Kanoulas E, de Rijke M, Fang H, Yilmaz E (eds) Proc. 3rd ACM SIGIR international conference on the theory of information retrieval (ICTIR 2017). ACM Press, New York, pp 67\u201374","DOI":"10.1145\/3121050.3121058"},{"issue":"3","key":"1_CR17","doi-asserted-by":"publisher","first-page":"409","DOI":"10.1109\/TKDE.2018.2840708","volume":"31","author":"M Ferrante","year":"2019","unstructured":"Ferrante M, Ferro N, Pontarollo S (2019) A general theory of IR evaluation measures. IEEE Trans Knowl Data Eng 31(3):409\u2013422","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"1_CR18","series-title":"Lecture notes in computer science (LNCS), vol","doi-asserted-by":"crossref","first-page":"552","DOI":"10.1007\/978-3-642-15754-7_68","volume-title":"Multilingual information access evaluation, vol. I text retrieval experiments \u2013 tenth workshop of the cross\u2013language evaluation forum (CLEF 2009). Revised selected papers","author":"N Ferro","year":"2010","unstructured":"Ferro N, Harman D (2010) CLEF 2009: Grid@CLEF pilot track overview. In: Peters C, Di Nunzio GM, Kurimo M, Mandl T, Mostefa D, Pe\u00f1as A, Roda G (eds) Multilingual information access evaluation, vol. I text retrieval experiments \u2013 tenth workshop of the cross\u2013language evaluation forum (CLEF 2009). Revised selected papers. Lecture notes in computer science (LNCS), vol 6241. Springer, Heidelberg, pp 552\u2013565"},{"key":"1_CR19","doi-asserted-by":"crossref","unstructured":"Ferro N, Silvello G (2016) A general linear mixed models approach to study system component effects. In: Perego R, Sebastiani F, Aslam J, Ruthven I, Zobel J (eds) Proc. 39th annual international ACM SIGIR conference on research and development in information retrieval (SIGIR 2016). ACM Press, New York, pp 25\u201334","DOI":"10.1145\/2911451.2911530"},{"issue":"2","key":"1_CR20","doi-asserted-by":"publisher","first-page":"187","DOI":"10.1002\/asi.23910","volume":"69","author":"N Ferro","year":"2017","unstructured":"Ferro N, Silvello G (2017) Towards an anatomy of IR system component performances. J Am Soc Inf Sci Technol 69(2):187\u2013200","journal-title":"J Am Soc Inf Sci Technol"},{"key":"1_CR21","first-page":"239","volume-title":"Lecture notes in computer science (LNCS), vol 11,018","author":"Ferro N, Maistro M, Sakai T, Soboroff I (2018) Overview of CENTRE@CLEF 2018: a first tale in the systematic reproducibility realm. In: Bellot P, Trabelsi C, Mothe J, Murtagh F, Nie JY, Soulier L, SanJuan E, Cappellato L, Ferro N (eds) Experimental IR meets multilinguality, multimodality, and interaction. Proceedings of the ninth international conference of the CLEF association (CLEF","year":"2018","unstructured":"Ferro N, Maistro M, Sakai T, Soboroff I (2018) Overview of CENTRE@CLEF 2018: a first tale in the systematic reproducibility realm. In: Bellot P, Trabelsi C, Mothe J, Murtagh F, Nie JY, Soulier L, SanJuan E, Cappellato L, Ferro N (eds) Experimental IR meets multilinguality, multimodality, and interaction. Proceedings of the ninth international conference of the CLEF association (CLEF 2018). Lecture notes in computer science (LNCS), vol 11,018. Springer, Heidelberg, pp 239\u2013246"},{"issue":"2","key":"1_CR22","doi-asserted-by":"publisher","first-page":"19","DOI":"10.1145\/2422256.2422259","volume":"46","author":"N Fuhr","year":"2012","unstructured":"Fuhr N (2012) Salton award lecture: information retrieval as engineering science. SIGIR Forum 46(2):19\u201328","journal-title":"SIGIR Forum"},{"key":"1_CR23","series-title":"Lecture notes in computer science (LNCS), vol","doi-asserted-by":"publisher","first-page":"124","DOI":"10.1007\/978-3-642-15998-5_14","volume-title":"Multilingual and multimodal information access evaluation. Proceedings of the international conference of the cross-language evaluation forum (CLEF 2010)","author":"A Hanbury","year":"2010","unstructured":"Hanbury A, M\u00fcller H (2010) Automated component-level evaluation: present and future. In: Agosti M, Ferro N, Peters C, de Rijke M, Smeaton A (eds) Multilingual and multimodal information access evaluation. Proceedings of the international conference of the cross-language evaluation forum (CLEF 2010). Lecture notes in computer science (LNCS), vol 6360. Springer, Heidelberg, pp 124\u2013135"},{"key":"1_CR24","doi-asserted-by":"publisher","DOI":"10.2200\/S00368ED1V01Y201105ICR019","volume-title":"Information retrieval evaluation","author":"DK Harman","year":"2011","unstructured":"Harman DK (2011) Information retrieval evaluation. Morgan & Claypool Publishers, San Rafael"},{"key":"1_CR25","volume-title":"TREC","year":"2005","unstructured":"Harman DK, Voorhees EM (eds) (2005) TREC. Experiment and evaluation in information retrieval. MIT Press, Cambridge"},{"key":"1_CR26","series-title":"Lecture notes in computer science (LNCS), vol","doi-asserted-by":"publisher","first-page":"7","DOI":"10.1007\/3-540-44645-1_2","volume-title":"Cross-language information retrieval and evaluation: workshop of cross-language evaluation forum (CLEF 2000)","author":"DK Harman","year":"2001","unstructured":"Harman DK, Braschler M, Hess M, Kluck M, Peters C, Sch\u00e4uble P, Sheridan P (2001) CLIR evaluation at TREC. In: Peters C (ed) Cross-language information retrieval and evaluation: workshop of cross-language evaluation forum (CLEF 2000). Lecture notes in computer science (LNCS), vol 2069. Springer, Heidelberg, pp 7\u201323"},{"issue":"4","key":"1_CR27","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3239570","volume":"10","author":"Frank Hopfgartner","year":"2018","unstructured":"Hopfgartner F, Hanbury A, M\u00fcller H, Eggel I, Balog K, Brodt T, Cormack GV, Lin J, Kalpathy-Cramer J, Kando N, Kato MP, Krithara A, Gollub T, Potthast M, Viegas E, Mercer S (2018) Evaluation-as-a-service for the computational sciences: overview and outlook. ACM J Data Inf Qual 10(4):15:1\u201315:32","journal-title":"Journal of Data and Information Quality"},{"key":"1_CR28","doi-asserted-by":"crossref","unstructured":"Hull DA (1993) Using statistical testing in the evaluation of retrieval experiments. In: Korfhage R, Rasmussen E, Willett P (eds) Proc. 16th annual international ACM SIGIR conference on research and development in information retrieval (SIGIR 1993). ACM Press, New York, pp 329\u2013338","DOI":"10.1145\/160688.160758"},{"issue":"4","key":"1_CR29","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin K, Kek\u00e4l\u00e4inen J (2002) Cumulated gain-based evaluation of IR techniques. ACM Trans Inf Syst 20(4):422\u2013446","journal-title":"ACM Trans Inf Syst"},{"key":"1_CR30","first-page":"361","volume-title":"Lecture notes in computer science (LNCS), vol 10,456","author":"Kanoulas E, Azzopardi L (2017) CLEF 2017 dynamic search evaluation lab overview. In: Jones GJF, Lawless S, Gonzalo J, Kelly L, Goeuriot L, Mandl T, Cappellato L, Ferro N (eds) Experimental IR meets multilinguality, multimodality, and interaction. Proceedings of the eighth international conference of the CLEF association (CLEF","year":"2017","unstructured":"Kanoulas E, Azzopardi L (2017) CLEF 2017 dynamic search evaluation lab overview. In: Jones GJF, Lawless S, Gonzalo J, Kelly L, Goeuriot L, Mandl T, Cappellato L, Ferro N (eds) Experimental IR meets multilinguality, multimodality, and interaction. Proceedings of the eighth international conference of the CLEF association (CLEF 2017). Lecture notes in computer science (LNCS), vol 10,456. Springer, Heidelberg, pp 361\u2013366"},{"issue":"13","key":"1_CR31","doi-asserted-by":"publisher","first-page":"1120","DOI":"10.1002\/asi.10137","volume":"53","author":"J Kek\u00e4l\u00e4inen","year":"2002","unstructured":"Kek\u00e4l\u00e4inen J, J\u00e4rvelin K (2002) Using graded relevance assessments in IR evaluation. J Am Soc Inf Sci Technol 53(13):1120\u20131129","journal-title":"J Am Soc Inf Sci Technol"},{"issue":"1\u20132","key":"1_CR32","first-page":"1","volume":"3","author":"D Kelly","year":"2009","unstructured":"Kelly D (2009) Methods for evaluating interactive information retrieval systems with users. Found Trends Inf Retr 3(1\u20132):1\u2013224","journal-title":"Found Trends Inf Retr"},{"key":"1_CR33","first-page":"239","volume-title":"Lecture Notes in Computer Science","author":"Andreas Lommatzsch","year":"2017","unstructured":"Lommatzsch A, Kille B, Hopfgartner F, Larson M, Brodt T, Seiler J, \u00d6zg\u00f6bek \u00d6 (2017) CLEF 2017 NewsREEL overview: a stream-based recommender task for evaluation and education. In: Jones GJF, Lawless S, Gonzalo J, Kelly L, Goeuriot L, Mandl T, Cappellato L, Ferro N (eds) Experimental IR meets multilinguality, multimodality, and interaction. Proceedings of the eighth international conference of the CLEF association (CLEF 2017). Lecture notes in computer science (LNCS), vol 10,456. Springer, Heidelberg, pp 239\u2013254"},{"issue":"1\u20132","key":"1_CR34","doi-asserted-by":"publisher","first-page":"73","DOI":"10.1023\/B:INRT.0000009441.78971.be","volume":"7","author":"P McNamee","year":"2004","unstructured":"McNamee P, Mayfield J (2004) Character N-gram tokenization for European language text retrieval. Inf Retr 7(1\u20132):73\u201397","journal-title":"Inf Retr"},{"issue":"9","key":"1_CR35","doi-asserted-by":"publisher","first-page":"810","DOI":"10.1002\/(SICI)1097-4571(199709)48:9<810::AID-ASI6>3.0.CO;2-U","volume":"48","author":"S Mizzaro","year":"1997","unstructured":"Mizzaro S (1997) Relevance: the whole history. J Am Soc Inf Sci Technol 48(9):810\u2013832","journal-title":"J Am Soc Inf Sci Technol"},{"issue":"1","key":"1_CR36","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/1416950.1416952","volume":"27","author":"Alistair Moffat","year":"2008","unstructured":"Moffat A, Zobel J (2008) Rank-biased precision for measurement of retrieval effectiveness. ACM Trans Inf Syst 27(1):2:1\u20132:27","journal-title":"ACM Transactions on Information Systems"},{"key":"1_CR37","unstructured":"Nardi A, Peters C, Vicedo JL, Ferro N (eds) (2006) CLEF 2006 working notes, CEUR workshop proceedings (CEUR-WS.org), ISSN 1613-0073. http:\/\/ceur-ws.org\/Vol-1172\/"},{"key":"1_CR38","unstructured":"Nardi A, Peters C, Ferro N (eds) (2007) CLEF 2007 working notes, CEUR workshop proceedings (CEUR-WS.org), ISSN 1613-0073. http:\/\/ceur-ws.org\/Vol-1173\/"},{"key":"1_CR39","first-page":"338","volume-title":"Lecture notes in computer science (LNCS), vol 10,456","author":"Pasi G, Jones GJF, Marrara S, Sanvitto C, Ganguly D, Sen P (2017) Overview of the CLEF 2017 personalised information retrieval pilot lab (PIR-CLEF 2017). In: Jones GJF, Lawless S, Gonzalo J, Kelly L, Goeuriot L, Mandl T, Cappellato L, Ferro N (eds) Experimental IR meets multilinguality, multimodality, and interaction. Proceedings of the eighth international conference of the CLEF association (CLEF","year":"2017","unstructured":"Pasi G, Jones GJF, Marrara S, Sanvitto C, Ganguly D, Sen P (2017) Overview of the CLEF 2017 personalised information retrieval pilot lab (PIR-CLEF 2017). In: Jones GJF, Lawless S, Gonzalo J, Kelly L, Goeuriot L, Mandl T, Cappellato L, Ferro N (eds) Experimental IR meets multilinguality, multimodality, and interaction. Proceedings of the eighth international conference of the CLEF association (CLEF 2017). Lecture notes in computer science (LNCS), vol 10,456. Springer, Heidelberg, pp 338\u2013345"},{"key":"1_CR40","doi-asserted-by":"crossref","unstructured":"Peters C (ed) (2001) Cross-language information retrieval and evaluation: workshop of cross-language evaluation forum (CLEF 2000). Lecture notes in computer science (LNCS), vol 2069. Springer, Heidelberg","DOI":"10.1007\/3-540-44645-1"},{"key":"1_CR41","unstructured":"Rowe BR, Wood DW, Link AL, Simoni DA (2010) Economic impact assessment of NIST\u2019s text retrieval conference (TREC) program. RTI Project Number 0211875, RTI International. http:\/\/trec.nist.gov\/pubs\/2010.economic.impact.pdf"},{"key":"1_CR42","doi-asserted-by":"crossref","unstructured":"Sakai T (2006) Evaluating evaluation metrics based on the bootstrap. In: Efthimiadis EN, Dumais S, Hawking D, J\u00e4rvelin K (eds) Proc. 29th annual international ACM SIGIR conference on research and development in information retrieval (SIGIR 2006). ACM Press, New York, pp 525\u2013532","DOI":"10.1145\/1148170.1148261"},{"key":"1_CR43","doi-asserted-by":"crossref","unstructured":"Sakai T (2012) Evaluation with informational and navigational intents. In: Mille A, Gandon FL, Misselis J, Rabinovich M, Staab S (eds) Proc. 21st international conference on world wide web (WWW 2012). ACM Press, New York, pp 499\u2013508","DOI":"10.1145\/2187836.2187904"},{"key":"1_CR44","series-title":"Lecture notes in computer science (LNCS), vol","doi-asserted-by":"crossref","first-page":"116","DOI":"10.1007\/978-3-642-54798-0_6","volume-title":"Bridging between information retrieval and databases - PROMISE winter school 2013, revised tutorial lectures","author":"T Sakai","year":"2014","unstructured":"Sakai T (2014a) Metrics, statistics, tests. In: Ferro N (ed) Bridging between information retrieval and databases - PROMISE winter school 2013, revised tutorial lectures. Lecture notes in computer science (LNCS), vol 8173. Springer, Heidelberg, pp 116\u2013163"},{"issue":"1","key":"1_CR45","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1145\/2641383.2641385","volume":"48","author":"T Sakai","year":"2014","unstructured":"Sakai T (2014b) Statistical reform in information retrieval? SIGIR Forum 48(1):3\u201312","journal-title":"SIGIR Forum"},{"issue":"4","key":"1_CR46","doi-asserted-by":"publisher","first-page":"247","DOI":"10.1561\/1500000009","volume":"4","author":"M Sanderson","year":"2010","unstructured":"Sanderson M (2010) Test collection based evaluation of information retrieval systems. Found Trends Inf Retr 4(4):247\u2013375","journal-title":"Found Trends Inf Retr"},{"issue":"6","key":"1_CR47","doi-asserted-by":"publisher","first-page":"321","DOI":"10.1002\/asi.4630260604","volume":"26","author":"T Saracevic","year":"1975","unstructured":"Saracevic T (1975) RELEVANCE: a review of and a framework for the thinking on the notion in information science. J Am Soc Inf Sci Technol 26(6):321\u2013343","journal-title":"J Am Soc Inf Sci Technol"},{"issue":"44","key":"1_CR48","doi-asserted-by":"publisher","first-page":"495","DOI":"10.1016\/S0306-4573(97)00027-7","volume":"33","author":"J Savoy","year":"1997","unstructured":"Savoy J (1997) Statistical inference in retrieval effectiveness evaluation. Inf Process Manag 33(44):495\u2013512","journal-title":"Inf Process Manag"},{"key":"1_CR49","volume-title":"Information retrieval experiment","year":"1981","unstructured":"Sp\u00e4rck Jones K (ed) (1981) Information retrieval experiment. Butterworths, London"},{"issue":"4","key":"1_CR50","doi-asserted-by":"publisher","first-page":"613","DOI":"10.1002\/asi.21494","volume":"62","author":"CV Thornley","year":"2011","unstructured":"Thornley CV, Johnson AC, Smeaton AF, Lee H (2011) The scholarly impact of TRECVid (2003\u20132009). J Am Soc Inf Sci Technol 62(4):613\u2013627","journal-title":"J Am Soc Inf Sci Technol"},{"key":"1_CR51","doi-asserted-by":"publisher","first-page":"95","DOI":"10.1007\/978-3-642-23708-9_12","volume-title":"Multilingual and Multimodal Information Access Evaluation","author":"Theodora Tsikrika","year":"2011","unstructured":"Tsikrika T, Garcia Seco de Herrera A, M\u00fcller H (2011) Assessing the scholarly impact of image CLEF. In: Forner P, Gonzalo J, Kek\u00e4l\u00e4inen J, Lalmas M, de Rijke M (eds) Multilingual and multimodal information access evaluation. Proceedings of the second international conference of the cross-language evaluation forum (CLEF 2011). Lecture notes in computer science (LNCS), vol 6941. Springer, Heidelberg, pp 95\u2013106"},{"key":"1_CR52","series-title":"Lecture notes in computer science (LNCS), vol","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/978-3-642-40802-1_1","volume-title":"Information access evaluation meets multilinguality, multimodality, and visualization. Proceedings of the fourth international conference of the CLEF initiative (CLEF 2013)","author":"T Tsikrika","year":"2013","unstructured":"Tsikrika T, Larsen B, M\u00fcller H, Endrullis S, Rahm E (2013) The scholarly impact of CLEF (2000\u20132009). In: Forner P, M\u00fcller H, Paredes R, Rosso P, Stein B (eds) Information access evaluation meets multilinguality, multimodality, and visualization. Proceedings of the fourth international conference of the CLEF initiative (CLEF 2013). Lecture notes in computer science (LNCS), vol 8138. Springer, Heidelberg, pp 1\u201312"},{"issue":"4","key":"1_CR53","doi-asserted-by":"publisher","first-page":"365","DOI":"10.1108\/eb026584","volume":"30","author":"CJ Rijsbergen van","year":"1974","unstructured":"van Rijsbergen CJ (1974) Foundations of evaluation. J Doc 30(4):365\u2013373","journal-title":"J Doc"},{"key":"1_CR54","unstructured":"Voorhees EM, Harman DK (1998) Overview of the seventh text retrieval conference (TREC-7). In: Voorhees EM, Harman DK (eds) The seventh text retrieval conference (TREC-7). National Institute of Standards and Technology (NIST), Special Publication 500-242, Washington, pp 1\u201324"},{"key":"1_CR55","doi-asserted-by":"crossref","unstructured":"Yilmaz E, Aslam JA (2006) Estimating average precision with incomplete and imperfect judgments. In: Yu PS, Tsotras V, Fox EA, Liu CB (eds) Proc. 15th international conference on information and knowledge management (CIKM 2006). ACM Press, New York, pp 102\u2013111","DOI":"10.1145\/1183614.1183633"}],"container-title":["The Information Retrieval Series","Information Retrieval Evaluation in a Changing World"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-22948-1_1","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,9,18]],"date-time":"2023-09-18T21:22:57Z","timestamp":1695072177000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-22948-1_1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019]]},"ISBN":["9783030229474","9783030229481"],"references-count":55,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-22948-1_1","relation":{},"ISSN":["1387-5264"],"issn-type":[{"value":"1387-5264","type":"print"}],"subject":[],"published":{"date-parts":[[2019]]},"assertion":[{"value":"14 August 2019","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}}]}}