{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,8]],"date-time":"2025-10-08T16:29:04Z","timestamp":1759940944573},"publisher-location":"Cham","reference-count":28,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783319445632"},{"type":"electronic","value":"9783319445649"}],"license":[{"start":{"date-parts":[[2016,1,1]],"date-time":"2016-01-01T00:00:00Z","timestamp":1451606400000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2016]]},"DOI":"10.1007\/978-3-319-44564-9_2","type":"book-chapter","created":{"date-parts":[[2016,8,22]],"date-time":"2016-08-22T02:00:36Z","timestamp":1471831236000},"page":"16-27","source":"Crossref","is-referenced-by-count":7,"title":["The CLEF Monolingual Grid of Points"],"prefix":"10.1007","author":[{"given":"Nicola","family":"Ferro","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Gianmaria","family":"Silvello","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2016,8,23]]},"reference":[{"issue":"2","key":"2_CR1","doi-asserted-by":"crossref","first-page":"107","DOI":"10.1145\/2888422.2888439","volume":"49","author":"J Arguello","year":"2015","unstructured":"Arguello, J., Crane, M., Diaz, F., Lin, J., Trotman, A.: Report on the SIGIR 2015 workshop on reproducibility, inexplicability, and generalizability of results (RIGOR). SIGIR Forum 49(2), 107\u2013116 (2015)","journal-title":"SIGIR Forum"},{"key":"2_CR2","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"crossref","first-page":"89","DOI":"10.1007\/3-540-44645-1_9","volume-title":"Cross-Language Information Retrieval and Evaluation","author":"M Braschler","year":"2001","unstructured":"Braschler, M.: CLEF 2000 - overview of results. In: Peters, C. (ed.) CLEF 2000. LNCS, vol. 2069, p. 89. Springer, Heidelberg (2001)"},{"key":"2_CR3","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"crossref","first-page":"9","DOI":"10.1007\/3-540-45691-0_2","volume-title":"Evaluation of Cross-Language Information Retrieval Systems","author":"M Braschler","year":"2002","unstructured":"Braschler, M.: CLEF 2001 - overview of results. In: Peters, C., Braschler, M., Gonzalo, J., Kluck, M. (eds.) CLEF 2001. LNCS, vol. 2406, pp. 9\u201326. Springer, Heidelberg (2002)"},{"key":"2_CR4","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"crossref","first-page":"9","DOI":"10.1007\/978-3-540-45237-9_2","volume-title":"Advances in Cross-Language Information Retrieval","author":"M Braschler","year":"2003","unstructured":"Braschler, M.: CLEF 2002 \u2013 overview of results. In: Peters, C., Braschler, M., Gonzalo, J. (eds.) CLEF 2002. LNCS, vol. 2785, pp. 9\u201327. Springer, Heidelberg (2003)"},{"key":"2_CR5","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"crossref","first-page":"44","DOI":"10.1007\/978-3-540-30222-3_5","volume-title":"Comparative Evaluation of Multilingual Information Access Systems","author":"M Braschler","year":"2004","unstructured":"Braschler, M.: CLEF 2003 \u2013 overview of results. In: Peters, C., Gonzalo, J., Braschler, M., Kluck, M. (eds.) CLEF 2003. LNCS, vol. 3237, pp. 44\u201363. Springer, Heidelberg (2004)"},{"key":"2_CR6","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"crossref","first-page":"10","DOI":"10.1007\/11519645_2","volume-title":"Multilingual Information Access for Text, Speech and Images","author":"M Braschler","year":"2005","unstructured":"Braschler, M., Di Nunzio, G.M., Ferro, N., Peters, C.: CLEF 2004: ad hoc track overview and results analysis. In: Peters, C., Clough, P., Gonzalo, J., Jones, G.J.F., Kluck, M., Magnini, B. (eds.) CLEF 2004. LNCS, vol. 3491, pp. 10\u201326. Springer, Heidelberg (2005)"},{"issue":"3\u20134","key":"2_CR7","doi-asserted-by":"crossref","first-page":"291","DOI":"10.1023\/B:INRT.0000011208.60754.a1","volume":"7","author":"M Braschler","year":"2004","unstructured":"Braschler, M., Ripplinger, B.: How effective is stemming and decompounding for german text retrieval? Inf. Retr. 7(3\u20134), 291\u2013316 (2004)","journal-title":"Inf. Retr."},{"key":"2_CR8","unstructured":"Buckley, C., Voorhees, E.M.: Retrieval system evaluation. In: TREC: Experiment and Evaluation in Information Retrieval, pp. 53\u201378. MIT Press (2005)"},{"key":"2_CR9","unstructured":"Burnham, K.P., Anderson, D.R.: Model Selection and Multimodel Inference: A Practical Information-Theoretic Approach, p. 488. Springer, Heidelberg (2002)"},{"key":"2_CR10","doi-asserted-by":"crossref","unstructured":"Chapelle, O., Metzler, D., Zhang, Y., Grinspan, P.: Expected reciprocal rank for graded relevance. In: Proceedings of 18th International Conference on Information and Knowledge Management (CIKM), pp. 621\u2013630. ACM Press (2009)","DOI":"10.1145\/1645953.1646033"},{"key":"2_CR11","unstructured":"Di Buccio, E., Di Nunzio, G.M., Ferro, N., Harman, D.K., Maistro, M., Silvello, G.: Unfolding off-the-shelf IR systems for reproducibility. In: Proceedings of SIGIR Workshop on Reproducibility, Inexplicability, and Generalizability of Results (RIGOR) (2015)"},{"key":"2_CR12","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"crossref","first-page":"11","DOI":"10.1007\/11878773_2","volume-title":"Accessing Multilingual Information Repositories","author":"GM Nunzio Di","year":"2006","unstructured":"Di Nunzio, G.M., Ferro, N., Jones, G.J.F., Peters, C.: CLEF 2005: ad hoc track overview. In: Peters, C., et al. (eds.) CLEF 2005. LNCS, vol. 4022, pp. 11\u201336. Springer, Heidelberg (2006)"},{"key":"2_CR13","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"crossref","first-page":"21","DOI":"10.1007\/978-3-540-74999-8_3","volume-title":"Evaluation of Multilingual and Multi-modal Information Retrieval","author":"GM Nunzio Di","year":"2007","unstructured":"Di Nunzio, G.M., Ferro, N., Mandl, T., Peters, C.: CLEF 2006: ad hoc track overview. In: Peters, C., et al. (eds.) CLEF 2006. LNCS, vol. 4730, pp. 21\u201334. Springer, Heidelberg (2007)"},{"issue":"1","key":"2_CR14","doi-asserted-by":"crossref","first-page":"68","DOI":"10.1145\/2964797.2964808","volume":"50","author":"N Ferro","year":"2016","unstructured":"Ferro, N., Fuhr, N., J\u00e4rvelin, K., Kando, N., Lippold, M., Zobel, J.: Increasing reproducibility in IR: findings from the Dagstuhl seminar on \u201creproducibility of data-oriented experiments in e-science\u201d. SIGIR Forum 50(1), 68\u201382 (2016)","journal-title":"SIGIR Forum"},{"key":"2_CR15","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"crossref","first-page":"552","DOI":"10.1007\/978-3-642-15754-7_68","volume-title":"Multilingual Information Access Evaluation I","author":"N Ferro","year":"2010","unstructured":"Ferro, N., Harman, D.: CLEF 2009: Grid@CLEF pilot track overview. In: Roda, G., Peters, C., Nunzio, G.M., Kurimo, M., Mandl, T., Mostefa, D., Pe\u00f1as, A. (eds.) CLEF 2009. LNCS, vol. 6241, pp. 552\u2013565. Springer, Heidelberg (2010)"},{"key":"2_CR16","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"crossref","first-page":"31","DOI":"10.1007\/978-3-319-11382-1_4","volume-title":"Information Access Evaluation","author":"N Ferro","year":"2014","unstructured":"Ferro, N., Silvello, G.: CLEF 15th birthday: what can we learn from ad hoc retrieval? In: Kanoulas, E., Lupu, M., Clough, P., Sanderson, M., Hall, M., Hanbury, A., Toms, E. (eds.) CLEF 2014. LNCS, vol. 8685, pp. 31\u201343. Springer, Heidelberg (2014)"},{"key":"2_CR17","doi-asserted-by":"crossref","unstructured":"Ferro, N., Silvello, G.: A general linear mixed models approach to study system component effects. In: Proceedings of 39th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR). ACM Press (2016)","DOI":"10.1145\/2911451.2911530"},{"issue":"4","key":"2_CR18","doi-asserted-by":"crossref","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin, K., Kek\u00e4l\u00e4inen, J.: Cumulated gain-based evaluation of IR techniques. ACM Trans. Inf. Syst. (TOIS) 20(4), 422\u2013446 (2002)","journal-title":"ACM Trans. Inf. Syst. (TOIS)"},{"key":"2_CR19","unstructured":"Kluck, M., Womser-Hacker, C.: Inside the evaluation process of the cross-language evaluation forum (CLEF): issues of multilingual topic creation and multilingual relevance assessment. In: Proceedings of 3rd International Language Resources and Evaluation Conference (LREC 2002) (2002)"},{"issue":"1","key":"2_CR20","doi-asserted-by":"crossref","first-page":"79","DOI":"10.1214\/aoms\/1177729694","volume":"22","author":"S Kullback","year":"1951","unstructured":"Kullback, S., Leibler, R.A.: On information and sufficiency. Ann. Math. Stat. 22(1), 79\u201386 (1951)","journal-title":"Ann. Math. Stat."},{"key":"2_CR21","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"408","DOI":"10.1007\/978-3-319-30671-1_30","volume-title":"Advances in Information Retrieval","author":"J Lin","year":"2016","unstructured":"Lin, J., et al.: Toward reproducible baselines: the open-source IR reproducibility challenge. In: Ferro, N., et al. (eds.) ECIR 2016. LNCS, vol. 9626, pp. 408\u2013420. Springer, Heidelberg (2016). doi: 10.1007\/978-3-319-30671-1_30"},{"key":"2_CR22","unstructured":"Macdonald, C., McCreadie, R., Santos, R.L.T., Ounis, I.: From puppy to maturity: experiences in developing terrier. In: Proceedings of OSIR at SIGIR, pp. 60\u201363 (2012)"},{"issue":"1","key":"2_CR23","doi-asserted-by":"crossref","first-page":"2:1","DOI":"10.1145\/1416950.1416952","volume":"27","author":"A Moffat","year":"2008","unstructured":"Moffat, A., Zobel, J.: Rank-biased precision for measurement of retrieval effectiveness. ACM Trans. Inf. Syst. (TOIS) 27(1), 2:1\u20132:27 (2008)","journal-title":"ACM Trans. Inf. Syst. (TOIS)"},{"key":"2_CR24","first-page":"9","volume-title":"Information Retrieval Experiment","author":"SE Robertson","year":"1981","unstructured":"Robertson, S.E.: The methodology of information retrieval experiment. In: Jones, K.S. (ed.) Information Retrieval Experiment, pp. 9\u201331. Butterworths, London (1981)"},{"issue":"4","key":"2_CR25","doi-asserted-by":"crossref","first-page":"247","DOI":"10.1561\/1500000009","volume":"4","author":"M Sanderson","year":"2010","unstructured":"Sanderson, M.: Test collection based evaluation of information retrieval systems. Found. Trends Inf. Retr. 4(4), 247\u2013375 (2010)","journal-title":"Found. Trends Inf. Retr."},{"issue":"2","key":"2_CR26","doi-asserted-by":"crossref","first-page":"95","DOI":"10.1145\/2422256.2422269","volume":"46","author":"A Trotman","year":"2012","unstructured":"Trotman, A., Clarke, C.L.A., Ounis, I., Culpepper, J.S., Cartright, M.A., Geva, S.: Open source information retrieval: a report on the SIGIR 2012 workshop. ACM SIGIR Forum 46(2), 95\u2013101 (2012)","journal-title":"ACM SIGIR Forum"},{"key":"2_CR27","doi-asserted-by":"crossref","DOI":"10.1007\/978-1-4899-4493-1","volume-title":"Kernel Smoothing","author":"MP Wand","year":"1995","unstructured":"Wand, M.P., Jones, M.C.: Kernel Smoothing. Chapman and Hall\/CRC, Boca Raton (1995)"},{"key":"2_CR28","doi-asserted-by":"crossref","unstructured":"Webber, W., Moffat, A., Zobel, J.: Score standardization for inter-collection comparison of retrieval systems. In: Proceedings of 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR), pp. 51\u201358. ACM Press (2008)","DOI":"10.1145\/1390334.1390346"}],"container-title":["Lecture Notes in Computer Science","Experimental IR Meets Multilinguality, Multimodality, and Interaction"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-319-44564-9_2","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2017,6,24]],"date-time":"2017-06-24T21:05:18Z","timestamp":1498338318000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-319-44564-9_2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016]]},"ISBN":["9783319445632","9783319445649"],"references-count":28,"URL":"https:\/\/doi.org\/10.1007\/978-3-319-44564-9_2","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2016]]}}}