{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,16]],"date-time":"2026-01-16T17:30:46Z","timestamp":1768584646891,"version":"3.49.0"},"publisher-location":"Cham","reference-count":28,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783319989310","type":"print"},{"value":"9783319989327","type":"electronic"}],"license":[{"start":{"date-parts":[[2018,1,1]],"date-time":"2018-01-01T00:00:00Z","timestamp":1514764800000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2018]]},"DOI":"10.1007\/978-3-319-98932-7_23","type":"book-chapter","created":{"date-parts":[[2018,8,14]],"date-time":"2018-08-14T11:51:14Z","timestamp":1534247474000},"page":"239-246","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":10,"title":["Overview of CENTRE@CLEF 2018: A\u00a0First Tale in the Systematic Reproducibility Realm"],"prefix":"10.1007","author":[{"given":"Nicola","family":"Ferro","sequence":"first","affiliation":[]},{"given":"Maria","family":"Maistro","sequence":"additional","affiliation":[]},{"given":"Tetsuya","family":"Sakai","sequence":"additional","affiliation":[]},{"given":"Ian","family":"Soboroff","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2018,8,15]]},"reference":[{"key":"23_CR1","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"15","DOI":"10.1007\/978-3-642-04447-2_2","volume-title":"Evaluating Systems for Multilingual and Multimodal Information Access","author":"E Agirre","year":"2009","unstructured":"Agirre, E., Di Nunzio, G.M., Ferro, N., Mandl, T., Peters, C.: CLEF 2008: ad hoc track overview. In: Peters, C., et al. (eds.) CLEF 2008. LNCS, vol. 5706, pp. 15\u201337. Springer, Heidelberg (2009). https:\/\/doi.org\/10.1007\/978-3-642-04447-2_2"},{"issue":"2","key":"23_CR2","doi-asserted-by":"publisher","first-page":"107","DOI":"10.1145\/2888422.2888439","volume":"49","author":"J Arguello","year":"2015","unstructured":"Arguello, J., Crane, M., Diaz, F., Lin, J., Trotman, A.: Report on the SIGIR 2015 workshop on reproducibility, inexplicability, and generalizability of results (RIGOR). SIGIR Forum 49(2), 107\u2013116 (2015)","journal-title":"SIGIR Forum"},{"key":"23_CR3","doi-asserted-by":"crossref","unstructured":"Armstrong, T.G., Moffat, A., Webber, W., Zobel, J.: Has adhoc retrieval improved since 1994? In: Allan, J., Aslam, J.A., Sanderson, M., Zhai, C., Zobel, J. (eds.) Proceedings of the 32nd Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2009), pp. 692\u2013693. ACM Press, New York (2009)","DOI":"10.1145\/1571941.1572081"},{"key":"23_CR4","doi-asserted-by":"crossref","unstructured":"Armstrong, T.G., Moffat, A., Webber, W., Zobel, J.: Improvements that don\u2019t add up: ad-hoc retrieval results since 1998. In: Cheung, D.W.L., Song, I.Y., Chu, W.W., Hu, X., Lin, J.J. (eds.) Proceedings of the 18th International Conference on Information and Knowledge Management (CIKM 2009), pp. 601\u2013610. ACM Press, New York (2009)","DOI":"10.1145\/1645953.1646031"},{"key":"23_CR5","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"44","DOI":"10.1007\/978-3-540-30222-3_5","volume-title":"Comparative Evaluation of Multilingual Information Access Systems","author":"M Braschler","year":"2004","unstructured":"Braschler, M.: CLEF 2003 \u2013 overview of results. In: Peters, C., Gonzalo, J., Braschler, M., Kluck, M. (eds.) CLEF 2003. LNCS, vol. 3237, pp. 44\u201363. Springer, Heidelberg (2004). https:\/\/doi.org\/10.1007\/978-3-540-30222-3_5"},{"key":"23_CR6","unstructured":"Clarke, C.L.A., Craswell, N., Voorhees, E.M.: Overview of the TREC 2012 web track. In: Voorhees, E.M., Buckland, L.P. (eds.) Proceedings of the Twenty-First Text REtrieval Conference (TREC 2012), pp. 1\u20138. National Institute of Standards and Technology (NIST), Special Publication 500\u2013298, Washington (2013)"},{"key":"23_CR7","doi-asserted-by":"crossref","unstructured":"Collins-Thompson, K., Diaz, F., Clarke, C.L.A., Voorhees, E.M.: TREC 2013 web track overview. In: Voorhees, E.M. (ed.) Proceedings of the Twenty-Second Text REtrieval Conference (TREC 2013). National Institute of Standards and Technology (NIST), Special Publication 500\u2013302, Washington (2014)","DOI":"10.6028\/NIST.SP.500-302.web-overview"},{"key":"23_CR8","doi-asserted-by":"crossref","unstructured":"Collins-Thompson, K., Macdonald, C., Bennett, P.N., Voorhees, E.M.: TREC 2014 web track overview. In: Voorhees, E.M., Ellis, A. (eds.) Proceedings of the Twenty-Third Text REtrieval Conference (TREC 2014). National Institute of Standards and Technology (NIST), Special Publication 500\u2013308, Washington (2015)","DOI":"10.6028\/NIST.SP.500-308.web-overview"},{"key":"23_CR9","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"11","DOI":"10.1007\/11878773_2","volume-title":"Accessing Multilingual Information Repositories","author":"GM Nunzio Di","year":"2006","unstructured":"Di Nunzio, G.M., Ferro, N., Jones, G.J.F., Peters, C.: CLEF 2005: ad hoc track overview. In: Peters, C., et al. (eds.) CLEF 2005. LNCS, vol. 4022, pp. 11\u201336. Springer, Heidelberg (2006). https:\/\/doi.org\/10.1007\/11878773_2"},{"issue":"2","key":"23_CR10","first-page":"8:1","volume":"8","author":"N Ferro","year":"2017","unstructured":"Ferro, N.: Reproducibility challenges in information retrieval evaluation. ACM J. Data Inf. Qual. (JDIQ) 8(2), 8:1\u20138:4 (2017)","journal-title":"ACM J. Data Inf. Qual. (JDIQ)"},{"issue":"1","key":"23_CR11","doi-asserted-by":"publisher","first-page":"68","DOI":"10.1145\/2964797.2964808","volume":"50","author":"N Ferro","year":"2016","unstructured":"Ferro, N., Fuhr, N., J\u00e4rvelin, K., Kando, N., Lippold, M., Zobel, J.: Increasing reproducibility in IR: findings from the Dagstuhl seminar on \u201creproducibility of data-oriented experiments in e-science\". SIGIR Forum 50(1), 68\u201382 (2016)","journal-title":"SIGIR Forum"},{"key":"23_CR12","doi-asserted-by":"crossref","unstructured":"Ferro, N., Kelly, D.: SIGIR initiative to implement ACM artifact review and badging. SIGIR Forum 52(1) (2018)","DOI":"10.1145\/3274784.3274786"},{"key":"23_CR13","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"13","DOI":"10.1007\/978-3-642-15754-7_2","volume-title":"Multilingual Information Access Evaluation I. Text Retrieval Experiments","author":"N Ferro","year":"2010","unstructured":"Ferro, N., Peters, C.: CLEF 2009 ad hoc track overview: TEL and Persian tasks. In: Peters, C., et al. (eds.) CLEF 2009. LNCS, vol. 6241, pp. 13\u201335. Springer, Heidelberg (2010). https:\/\/doi.org\/10.1007\/978-3-642-15754-7_2"},{"key":"23_CR14","unstructured":"Freire, J., Fuhr, N., Rauber, A. (eds.): Report from Dagstuhl seminar 16041: reproducibility of data-oriented experiments in e-science. Dagstuhl Reports, vol. 6, no 1. Schloss Dagstuhl-Leibniz-Zentrum f\u00fcr Informatik, Germany (2016)"},{"key":"23_CR15","unstructured":"Gallagher, L., Mackenzie, J., Benham, R., Chen, R.C., Scholer, F., Culpepper, J.S.: RMIT at the NTCIR-13 we want web task. In: Kando, N., Fujita, S., Kato, M.P., Manabe, T. (eds.) Proceedings of the 13th NTCIR Conference on Evaluation of Information Access Technologies, pp. 402\u2013406. National Institute of Informatics, Tokyo (2017)"},{"key":"23_CR16","unstructured":"Guyot, J., Radhouani, S., Falquet, G.: Ontology-based multilingual information retrieval. In: Peters, C., Quochi, V., Ferro, N. (eds.) CLEF 2005 Working Notes. CEUR Workshop Proceedings. CEUR-WS.org (2005). http:\/\/ceur-ws.org\/Vol-1171\/ . ISSN 1613\u20130073"},{"key":"23_CR17","unstructured":"Jungwirth, M., Hanbury, A.: Replicating an experiment in cross-lingual information retrieval with explicit semantic analysis. In: Cappellato, L., Ferro, N., Nie, J.Y., Soulier, L. (eds.) CLEF 2018 Working Notes. CEUR Workshop Proceedings. CEUR-WS.org (2018). ISSN 1613\u20130073"},{"key":"23_CR18","volume-title":"Rank Correlation Methods","author":"MG Kendall","year":"1948","unstructured":"Kendall, M.G.: Rank Correlation Methods. Griffin, Oxford (1948)"},{"key":"23_CR19","volume-title":"Mathematics of Statistics - Part One","author":"JF Kenney","year":"1954","unstructured":"Kenney, J.F., Keeping, E.S.: Mathematics of Statistics - Part One, 3rd edn. D. Van Nostrand Company, Princeton (1954)","edition":"3"},{"issue":"4","key":"23_CR20","doi-asserted-by":"publisher","first-page":"23:1","DOI":"10.1145\/2882782","volume":"34","author":"S Kharazmi","year":"2016","unstructured":"Kharazmi, S., Scholer, F., Vallet, D., Sanderson, M.: Examining additivity and weak baselines. ACM Trans. Inf. Syst. (TOIS) 34(4), 23:1\u201323:18 (2016)","journal-title":"ACM Trans. Inf. Syst. (TOIS)"},{"key":"23_CR21","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"408","DOI":"10.1007\/978-3-319-30671-1_30","volume-title":"Advances in Information Retrieval","author":"J Lin","year":"2016","unstructured":"Lin, J., et al.: Toward reproducible baselines: the open-source IR reproducibility challenge. In: Ferro, N., et al. (eds.) ECIR 2016. LNCS, vol. 9626, pp. 408\u2013420. Springer, Cham (2016). https:\/\/doi.org\/10.1007\/978-3-319-30671-1_30"},{"key":"23_CR22","unstructured":"Luo, C., Sakai, T., Liu, Y., Dou, Z., Xiong, C., Xu, J.: Overview of the NTCIR-13 we want web task. In: Kando, N., Fujita, S., Kato, M.P., Manabe, T. (eds.) Proceedings of the 13th NTCIR Conference on Evaluation of Information Access Technologies, pp. 394\u2013401. National Institute of Informatics, Tokyo (2017)"},{"key":"23_CR23","doi-asserted-by":"crossref","unstructured":"McCreadie, R., et al.: University of Glasgow at TREC 2014: experiments with terrier in contextual suggestion, temporal summarisation and web tracks. In: Voorhees, E.M., Ellis, A. (eds.) Proceedings of the Twenty-Third Text REtrieval Conference (TREC 2014). National Institute of Standards and Technology (NIST), Special Publication 500\u2013308, Washington (2015)","DOI":"10.6028\/NIST.SP.500-308.web-uogTr"},{"key":"23_CR24","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"58","DOI":"10.1007\/978-3-642-04447-2_6","volume-title":"Evaluating Systems for Multilingual and Multimodal Information Access","author":"D Nguyen","year":"2009","unstructured":"Nguyen, D., Overwijk, A., Hauff, C., Trieschnigg, D.R.B., Hiemstra, D., de Jong, F.: WikiTranslate: query translation for cross-lingual information retrieval using only Wikipedia. In: Peters, C., et al. (eds.) CLEF 2008. LNCS, vol. 5706, pp. 58\u201365. Springer, Heidelberg (2009). https:\/\/doi.org\/10.1007\/978-3-642-04447-2_6"},{"key":"23_CR25","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"64","DOI":"10.1007\/978-3-540-30222-3_6","volume-title":"Comparative Evaluation of Multilingual Information Access Systems","author":"J Savoy","year":"2004","unstructured":"Savoy, J.: Report on CLEF-2003 multilingual tracks. In: Peters, C., Gonzalo, J., Braschler, M., Kluck, M. (eds.) CLEF 2003. LNCS, vol. 3237, pp. 64\u201373. Springer, Heidelberg (2004). https:\/\/doi.org\/10.1007\/978-3-540-30222-3_6"},{"key":"23_CR26","unstructured":"Sorg, P., Cimiano, P.: Cross-lingual information retrieval with explicit semantic analysis. In: Borri, F., Nardi, A., Peters, C., Ferro, N. (eds.) CLEF 2008 Working Notes. CEUR Workshop Proceedings. CEUR-WS.org (2008). http:\/\/ceur-ws.org\/Vol-1174\/ . ISSN 1613\u20130073"},{"key":"23_CR27","unstructured":"Yang, P., Fang, H.: Evaluating the effectiveness of axiomatic approaches in web track. In: Voorhees, E.M. (ed.) Proceedings of the Twenty-Second Text REtrieval Conference (TREC 2013). National Institute of Standards and Technology (NIST), Special Publication 500\u2013302, Washington (2014)"},{"key":"23_CR28","doi-asserted-by":"crossref","unstructured":"Zobel, J., Webber, W., Sanderson, M., Moffat, A.: Principles for robust evaluation infrastructure. In: Agosti, M., Ferro, N., Thanos, C. (eds.) Proceedings of the Workshop on Data infrastructurEs for Supporting Information Retrieval Evaluation (DESIRE 2011), pp. 3\u20136. ACM Press, New York (2011)","DOI":"10.1145\/2064227.2064247"}],"container-title":["Lecture Notes in Computer Science","Experimental IR Meets Multilinguality, Multimodality, and Interaction"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-319-98932-7_23","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,9]],"date-time":"2024-07-09T06:31:32Z","timestamp":1720506692000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-319-98932-7_23"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018]]},"ISBN":["9783319989310","9783319989327"],"references-count":28,"URL":"https:\/\/doi.org\/10.1007\/978-3-319-98932-7_23","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2018]]}}}