{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,9,13]],"date-time":"2024-09-13T07:51:07Z","timestamp":1726213867790},"publisher-location":"Cham","reference-count":21,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030229474"},{"type":"electronic","value":"9783030229481"}],"license":[{"start":{"date-parts":[[2019,1,1]],"date-time":"2019-01-01T00:00:00Z","timestamp":1546300800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2019]]},"DOI":"10.1007\/978-3-030-22948-1_23","type":"book-chapter","created":{"date-parts":[[2019,8,13]],"date-time":"2019-08-13T01:02:59Z","timestamp":1565658179000},"page":"555-564","update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Reproducibility and Validity in CLEF"],"prefix":"10.1007","author":[{"given":"Norbert","family":"Fuhr","sequence":"first","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,8,14]]},"reference":[{"key":"23_CR1","series-title":"Lecture notes in computer science (LNCS), vol","doi-asserted-by":"publisher","first-page":"88","DOI":"10.1007\/978-3-642-33247-0_11","volume-title":"Information access evaluation. Multilinguality, multimodality, and visual analytics. Proceedings of the third international conference of the CLEF initiative (CLEF 2012)","author":"M Agosti","year":"2012","unstructured":"Agosti M, Di Buccio E, Ferro N, Masiero I, Peruzzo S, Silvello G (2012) DIRECTions: design and specification of an IR evaluation infrastructure. In: Catarci T, Forner P, Hiemstra D, Pe\u00f1as A, Santucci G (eds) Information access evaluation. Multilinguality, multimodality, and visual analytics. Proceedings of the third international conference of the CLEF initiative (CLEF 2012). Lecture notes in computer science (LNCS), vol 7488. Springer, Heidelberg, pp 88\u201399"},{"key":"23_CR2","isbn-type":"print","doi-asserted-by":"publisher","first-page":"1081","DOI":"10.1145\/2911451.2911462","volume-title":"Proceedings of the 39th international ACM SIGIR conference on research and development in information retrieval, SIGIR 2016, Pisa, July 17\u201321, 2016","author":"M Angelini","year":"2016","unstructured":"Angelini M, Ferro N, Santucci G, Silvello G (2016) A visual analytics approach for what-if analysis of information retrieval systems. In Perego R, Sebastiani F, Aslam JA, Ruthven I, Zobel J (eds) Proceedings of the 39th international ACM SIGIR conference on research and development in information retrieval, SIGIR 2016, Pisa, July 17\u201321, 2016. ACM, New York, pp 1081\u20131084. ISBN 978-1-4503-4069-4. \n                    http:\/\/doi.acm.org\/10.1145\/2911451.2911462","ISBN":"http:\/\/id.crossref.org\/isbn\/9781450340694"},{"key":"23_CR3","isbn-type":"print","doi-asserted-by":"publisher","first-page":"601","DOI":"10.1145\/1645953.1646031","volume-title":"Proceedings of the 18th ACM conference on Information and knowledge management CIKM","author":"TG Armstrong","year":"2009","unstructured":"Armstrong TG, Moffat A, Webber W, Zobel J (2009) Improvements that don\u2019t add up: ad-hoc retrieval results since 1998. In: Cheung DW-L, Song I-Y, Chu WW, Hu X, Lin JJ (eds) Proceedings of the 18th ACM conference on Information and knowledge management CIKM. ACM, New York, pp 601\u2013610. ISBN 978-1-60558-512-3","ISBN":"http:\/\/id.crossref.org\/isbn\/9781605585123"},{"key":"23_CR4","series-title":"Lecture notes in computer science (LNCS), vol","doi-asserted-by":"crossref","first-page":"342","DOI":"10.1007\/978-3-642-15754-7_41","volume-title":"Multilingual information access evaluation vol. I. Text retrieval experiments \u2013 tenth workshop of the cross\u2013language evaluation forum (CLEF 2009). Revised selected papers","author":"R Besan\u00e7on","year":"2010","unstructured":"Besan\u00e7on R, Chaudiron S, Mostefa D, Timimi I, Choukri K, La\u00efb M (2010) Information filtering evaluation: overview of CLEF 2009 INFILE track. In: Peters C, Di Nunzio GM, Kurimo M, Mandl T, Mostefa D, Pe\u00f1as A, Roda G (eds) Multilingual information access evaluation vol. I. Text retrieval experiments \u2013 tenth workshop of the cross\u2013language evaluation forum (CLEF 2009). Revised selected papers. Lecture notes in computer science (LNCS), vol 6241. Springer, Heidelberg, pp 342\u2013353"},{"key":"23_CR5","series-title":"Lecture notes in computer science (LNCS), vol","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1007\/3-540-45691-0_2","volume-title":"Evaluation of cross-language information retrieval systems: second workshop of the cross\u2013language evaluation forum (CLEF 2001) revised papers","author":"M Braschler","year":"2002","unstructured":"Braschler M (2002) CLEF 2001 \u2013 overview of results. In: Peters C, Braschler M, Gonzalo J, Kluck M (eds) Evaluation of cross-language information retrieval systems: second workshop of the cross\u2013language evaluation forum (CLEF 2001) revised papers. Lecture notes in computer science (LNCS), vol 2406. Springer, Heidelberg, pp 9\u201326"},{"issue":"1","key":"23_CR6","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/2094072.2094076","volume":"30","author":"Benjamin A. Carterette","year":"2012","unstructured":"Carterette BA (2012) Multiple testing in statistical analysis of systems-based information retrieval experiments. ACM Trans Inf Syst 30(1):4:1\u20134:34. \n                    http:\/\/doi.acm.org\/10.1145\/2094072.2094076","journal-title":"ACM Transactions on Information Systems"},{"key":"23_CR7","doi-asserted-by":"publisher","first-page":"483","DOI":"10.1007\/11551362_46","volume-title":"Research and advanced technology for digital libraries, 9th European conference, ECDL 2005, Vienna, Austria, September 18\u201323, 2005, proceedings","author":"GM Nunzio Di","year":"2005","unstructured":"Di Nunzio GM, Ferro N (2005) DIRECT: a system for evaluating information access components of digital libraries. In: Rauber A, Christodoulakis C, Tjoa AM (eds) Research and advanced technology for digital libraries, 9th European conference, ECDL 2005, Vienna, Austria, September 18\u201323, 2005, proceedings. Springer, Berlin, pp 483\u2013484. \n                    https:\/\/doi.org\/10.1007\/11551362_46"},{"issue":"1","key":"23_CR8","doi-asserted-by":"publisher","first-page":"68","DOI":"10.1145\/2964797.2964808","volume":"50","author":"N Ferro","year":"2016","unstructured":"Ferro N, Fuhr N, Jarvelin K, Kando N, Lippold M, Zobel J (2016) Increasing reproducibility in IR: findings from the Dagstuhl seminar on \u201creproducibility of data-oriented experiments in e-science\u201d. SIGIR Forum 50(1):68\u201382. \n                    http:\/\/sigir.org\/files\/forum\/2016J\/p068.pdf","journal-title":"SIGIR Forum"},{"issue":"1","key":"23_CR9","doi-asserted-by":"publisher","first-page":"91","DOI":"10.1145\/3274784.3274789","volume":"52","author":"N Ferro","year":"2018","unstructured":"Ferro N, Fuhr N, Grefenstette G, Konstan JA, Castells P, Daly EM, Declerck T, Ekstrand MD, Geyer W, Gonzalo J, Kuflik T, Linden K, Magnini B, Nie J-Y, Perego R, Shapira B, Soboroff I, Tintarev N, Verspoor K, Willemsen MC, Zobel J (2018) The Dagstuhl perspectives workshop on performance modeling and prediction. SIGIR Forum 52(1):91\u2013101","journal-title":"SIGIR Forum"},{"issue":"1","key":"23_CR10","first-page":"108","volume":"6","author":"J Freire","year":"2016","unstructured":"Freire J, Fuhr N, Rauber A (2016) Reproducibility of data-oriented experiments in e-science. Dagstuhl Rep 6(1):108\u2013159. \n                    http:\/\/drops.dagstuhl.de\/opus\/institut_dagrep.php?fakultaet=07","journal-title":"Dagstuhl Rep"},{"issue":"3","key":"23_CR11","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1145\/3190580.3190586","volume":"51","author":"N Fuhr","year":"2017","unstructured":"Fuhr N (2017) Some common mistakes in ir evaluation, and how they can be avoided. SIGIR Forum 51(3):32\u201341. \n                    http:\/\/sigir.org\/wp-content\/uploads\/2018\/01\/p032.pdf","journal-title":"SIGIR Forum"},{"key":"23_CR12","series-title":"Lecture notes in computer science (LNCS), vol","doi-asserted-by":"publisher","first-page":"310","DOI":"10.1007\/11519645_32","volume-title":"Multilingual information access for text, speech and images: fifth workshop of the cross\u2013language evaluation forum (CLEF 2004) revised selected papers","author":"J Gonzalo","year":"2005","unstructured":"Gonzalo J, Oard DW (2005) iCLEF 2004 track overview: pilot experiments in interactive cross-language question answering. In: Peters C, Clough P, Gonzalo J, Jones GJF, Kluck M, Magnini B (eds) Multilingual information access for text, speech and images: fifth workshop of the cross\u2013language evaluation forum (CLEF 2004) revised selected papers. Lecture notes in computer science (LNCS), vol 3491. Springer, Heidelberg, pp 310\u2013322"},{"key":"23_CR13","unstructured":"Kille B, Lommatzsch A, Hopfgartner F, Larson M, Brodt T (2017) CLEF 2017 newsreel overview: offline and online evaluation of stream-based news recommender systems. In Cappellato L, Ferro N, Goeuriot L, Mandl T (eds) Working notes of CLEF 2017 - conference and labs of the evaluation forum, Dublin, September 11\u201314, 2017. CEUR workshop proceedings (CEUR-WS.org). ISSN 1613-0073. \n                    http:\/\/ceur-ws.org\/Vol-1866\/\n                    \n                  . \n                    http:\/\/ceur-ws.org\/Vol-1866\/invited_paper_17.pdf"},{"issue":"6251","key":"23_CR14","doi-asserted-by":"publisher","first-page":"943","DOI":"10.1126\/science.aac4716","volume":"349","author":"Open Science Collaboration","year":"2015","unstructured":"Open Science Collaboration (2015) Estimating the reproducibility of psychological science. Science 349(6251):943\u2013952","journal-title":"Science"},{"key":"23_CR15","unstructured":"Potthast M, Gollub T, Hagen M, Kiesel J, Michel M, Oberl\u00e4nder A, Tippmann M, Barr\u00f3n-Cede\u00f1o A, Gupta P, Rosso P, Stein B (2012) Overview of the 4th international competition on plagiarism detection. In: Forner P, Karlgren J, Womser-Hacker C, Ferro N (eds) CLEF 2012 working notes. CEUR workshop proceedings (CEUR-WS.org). ISSN 1613-0073, \n                    http:\/\/ceur-ws.org\/Vol-1178\/"},{"key":"23_CR16","unstructured":"Potthast M, Hagen M, Gollub T, Tippmann M, Kiesel J, Rosso P, Stamatatos E, Stein B (2013) Overview of the 5th international competition on plagiarism detection. In: Forner P, Navigli R, Tufis D, Ferro N (eds) CLEF 2013 working notes. CEUR workshop proceedings (CEUR-WS.org). ISSN 1613-0073. \n                    http:\/\/ceur-ws.org\/Vol-1179\/"},{"key":"23_CR17","first-page":"755","volume-title":"Lecture Notes in Computer Science","author":"Jinfeng Rao","year":"2015","unstructured":"Rao J, Lin JJ, Efron M (2015) Reproducible experiments on lexical and temporal feedback for tweet search. In Hanbury A, Kazai G, Rauber A, Fuhr N (eds) Advances in information retrieval - 37th European conference on IR research, ECIR 2015, Vienna, March 29\u2013April 2, 2015. Proceedings. Lecture Notes in Computer Science, vol 9022, pp 755\u2013767. ISBN 978-3-319-16353-6. \n                    https:\/\/doi.org\/10.1007\/978-3-319-16354-3_82"},{"key":"23_CR18","series-title":"Lecture notes in computer science (LNCS), vol","doi-asserted-by":"publisher","first-page":"484","DOI":"10.1007\/978-3-319-24027-5_47","volume-title":"Experimental IR meets multilinguality, multimodality, and interaction. Proceedings of the sixth international conference of the CLEF association (CLEF 2015)","author":"A Schuth","year":"2015","unstructured":"Schuth A, Balog K, Kelly L (2015) Overview of the living labs for information retrieval evaluation (LL4IR) CLEF Lab 2015. In: Mothe J, Savoy J, Kamps J, Pinel-Sauvagnat K, Jones GJF, SanJuan E, Cappellato L, Ferro N (eds) Experimental IR meets multilinguality, multimodality, and interaction. Proceedings of the sixth international conference of the CLEF association (CLEF 2015). Lecture notes in computer science (LNCS), vol 9283. Springer, Heidelberg, pp 484\u2013496"},{"issue":"2","key":"23_CR19","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1007\/s00799-016-0172-8","volume":"18","author":"G Silvello","year":"2017","unstructured":"Silvello G, Bordea G, Ferro N, Buitelaar P, Bogers T (2017) Semantic representation and enrichment of information retrieval experimental data. Int J Digit Libr 18(2):145\u2013172. ISSN 1432-5012. \n                    https:\/\/doi.org\/10.1007\/s00799-016-0172-8","journal-title":"Int J Digit Libr"},{"key":"23_CR20","isbn-type":"print","doi-asserted-by":"publisher","first-page":"316","DOI":"10.1145\/564376.564432","volume-title":"Proceedings of the 25th annual international ACM SIGIR conference on research and development in information retrieval, SIGIR \u201902","author":"EM Voorhees","year":"2002","unstructured":"Voorhees EM, Buckley C (2002) The effect of topic set size on retrieval experiment error. In: Proceedings of the 25th annual international ACM SIGIR conference on research and development in information retrieval, SIGIR \u201902. ACM, New York, pp 316\u2013323. ISBN 1-58113-561-0. \n                    https:\/\/doi.org\/10.1145\/564376.564432","ISBN":"http:\/\/id.crossref.org\/isbn\/1581135610"},{"key":"23_CR21","doi-asserted-by":"publisher","first-page":"531","DOI":"10.1016\/B978-0-12-374856-0.00015-8","volume-title":"Data Mining: Practical Machine Learning Tools and Techniques","author":"Ian H. Witten","year":"2011","unstructured":"Witten IH, Frank E, Hall MA (2011) Data mining: practical machine learning tools and techniques, 3rd edn. Morgan Kaufmann, San Francisco. ISBN 0123748569, ISBN 9780123748560"}],"container-title":["The Information Retrieval Series","Information Retrieval Evaluation in a Changing World"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-22948-1_23","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,8,13]],"date-time":"2019-08-13T01:09:59Z","timestamp":1565658599000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-22948-1_23"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019]]},"ISBN":["9783030229474","9783030229481"],"references-count":21,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-22948-1_23","relation":{},"ISSN":["1387-5264"],"issn-type":[{"type":"print","value":"1387-5264"}],"subject":[],"published":{"date-parts":[[2019]]},"assertion":[{"value":"14 August 2019","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}}]}}