{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,26]],"date-time":"2025-03-26T16:59:54Z","timestamp":1743008394768,"version":"3.40.3"},"publisher-location":"Cham","reference-count":29,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031717352"},{"type":"electronic","value":"9783031717369"}],"license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024]]},"DOI":"10.1007\/978-3-031-71736-9_16","type":"book-chapter","created":{"date-parts":[[2024,9,13]],"date-time":"2024-09-13T07:03:25Z","timestamp":1726211005000},"page":"215-226","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Replicability Measures for\u00a0Longitudinal Information Retrieval Evaluation"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9392-8646","authenticated-orcid":false,"given":"J\u00fcri","family":"Keller","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1765-2449","authenticated-orcid":false,"given":"Timo","family":"Breuer","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8817-4632","authenticated-orcid":false,"given":"Philipp","family":"Schaer","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,9,14]]},"reference":[{"key":"16_CR1","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"440","DOI":"10.1007\/978-3-031-42448-9_28","volume-title":"CLEF 2023","author":"R Alkhalifa","year":"2023","unstructured":"Alkhalifa, R., et al.: Overview of the CLEF-2023 LongEval lab on longitudinal evaluation of model performance. In: Arampatzis, A., et al. (eds.) CLEF 2023. LNCS, vol. 14163, pp. 440\u2013458. Springer, Cham (2023). https:\/\/doi.org\/10.1007\/978-3-031-42448-9_28"},{"key":"16_CR2","doi-asserted-by":"crossref","unstructured":"Alkhalifa, R., et al.: Extended overview of the CLEF-2023 LongEval lab on longitudinal evaluation of model performance. In: CLEF (Working Notes). CEUR Workshop Proceedings, vol.\u00a03497, pp. 2181\u20132203. CEUR-WS.org (2023)","DOI":"10.1007\/978-3-031-42448-9_28"},{"key":"16_CR3","unstructured":"Amati, G.: Probability models for information retrieval based on divergence from randomness. Ph.D. thesis, University of Glasgow, UK (2003)"},{"key":"16_CR4","unstructured":"Bar-Ilan, J.: Criteria for evaluating information retrieval systems in highly dynamic environments. In: WebDyn@WWW. CEUR Workshop Proceedings, vol.\u00a0702, pp. 70\u201377. CEUR-WS.org (2002)"},{"key":"16_CR5","doi-asserted-by":"crossref","unstructured":"Breuer, T., et al.: How to measure the reproducibility of system-oriented IR experiments. In: SIGIR, pp. 349\u2013358. ACM (2020)","DOI":"10.1145\/3397271.3401036"},{"key":"16_CR6","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"481","DOI":"10.1007\/978-3-030-72240-1_51","volume-title":"Advances in Information Retrieval","author":"T Breuer","year":"2021","unstructured":"Breuer, T., Ferro, N., Maistro, M., Schaer, P.: repro_eval: a Python interface to reproducibility measures of system-oriented IR experiments. In: Hiemstra, D., Moens, M.-F., Mothe, J., Perego, R., Potthast, M., Sebastiani, F. (eds.) ECIR 2021. LNCS, vol. 12657, pp. 481\u2013486. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-72240-1_51"},{"key":"16_CR7","doi-asserted-by":"crossref","unstructured":"Buckley, C., Voorhees, E.M.: Retrieval evaluation with incomplete information. In: SIGIR, pp. 25\u201332. ACM (2004)","DOI":"10.1145\/1008992.1009000"},{"key":"16_CR8","doi-asserted-by":"crossref","unstructured":"Chapelle, O., Zhang, Y.: A dynamic Bayesian network click model for web search ranking. In: WWW, pp. 1\u201310. ACM (2009)","DOI":"10.1145\/1526709.1526711"},{"key":"16_CR9","doi-asserted-by":"crossref","unstructured":"Clarke, C.L.A., et al.: Novelty and diversity in information retrieval evaluation. In: SIGIR, pp. 659\u2013666. ACM (2008)","DOI":"10.1145\/1390334.1390446"},{"key":"16_CR10","doi-asserted-by":"crossref","unstructured":"Cormack, G.V., Clarke, C.L.A., B\u00fcttcher, S.: Reciprocal rank fusion outperforms condorcet and individual rank learning methods. In: SIGIR, pp. 758\u2013759. ACM (2009)","DOI":"10.1145\/1571941.1572114"},{"key":"16_CR11","doi-asserted-by":"crossref","unstructured":"Craswell, N., Zoeter, O., Taylor, M.J., Ramsey, B.: An experimental comparison of click position-bias models. In: WSDM, pp. 87\u201394. ACM (2008)","DOI":"10.1145\/1341531.1341545"},{"key":"16_CR12","doi-asserted-by":"crossref","unstructured":"Dumais, S.T.: Temporal dynamics and information retrieval. In: CIKM, pp.\u00a07\u20138. ACM (2010)","DOI":"10.1145\/1871437.1871442"},{"key":"16_CR13","doi-asserted-by":"crossref","unstructured":"Dumais, S.T.: Putting searchers into search. In: SIGIR, pp.\u00a01\u20132. ACM (2014)","DOI":"10.1145\/2600428.2617557"},{"key":"16_CR14","doi-asserted-by":"crossref","unstructured":"Galusc\u00e1kov\u00e1, P., et al.: LongEval-retrieval: French-English dynamic test collection for continuous web search evaluation. In: SIGIR, pp. 3086\u20133094. ACM (2023)","DOI":"10.1145\/3539618.3591921"},{"key":"16_CR15","series-title":"The Information Retrieval Series","doi-asserted-by":"publisher","first-page":"511","DOI":"10.1007\/978-3-030-22948-1_21","volume-title":"Information Retrieval Evaluation in a Changing World","author":"F Hopfgartner","year":"2019","unstructured":"Hopfgartner, F., et al.: Continuous evaluation of large-scale information access systems: a case for living labs. In: Ferro, N., Peters, C. (eds.) Information Retrieval Evaluation in a Changing World. TIRS, vol. 41, pp. 511\u2013543. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-22948-1_21"},{"issue":"4","key":"16_CR16","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin, K., Kek\u00e4l\u00e4inen, J.: Cumulated gain-based evaluation of IR techniques. ACM Trans. Inf. Syst. 20(4), 422\u2013446 (2002)","journal-title":"ACM Trans. Inf. Syst."},{"issue":"1","key":"16_CR17","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/1292591.1292592","volume":"26","author":"EC Jensen","year":"2007","unstructured":"Jensen, E.C., Beitzel, S.M., Chowdhury, A., Frieder, O.: Repeatable evaluation of search services in dynamic environments. ACM Trans. Inf. Syst. 26(1), 1 (2007)","journal-title":"ACM Trans. Inf. Syst."},{"key":"16_CR18","unstructured":"Keller, J., Breuer, T., Schaer, P.: Evaluating temporal persistence using replicability measures. In: CLEF (Working Notes). CEUR Workshop Proceedings, vol.\u00a03497, pp. 2441\u20132457. CEUR-WS.org (2023)"},{"key":"16_CR19","doi-asserted-by":"crossref","unstructured":"Keller, J., Breuer, T., Schaer, P.: Evaluation of temporal change in IR test collections. In: ICTIR. ACM (2024)","DOI":"10.1145\/3664190.3672530"},{"key":"16_CR20","doi-asserted-by":"crossref","unstructured":"Khattab, O., Zaharia, M.: ColBERT: efficient and effective passage search via contextualized late interaction over BERT. In: SIGIR, pp. 39\u201348. ACM (2020)","DOI":"10.1145\/3397271.3401075"},{"issue":"3","key":"16_CR21","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2023.103332","volume":"60","author":"M Maistro","year":"2023","unstructured":"Maistro, M., Breuer, T., Schaer, P., Ferro, N.: An in-depth investigation on the behavior of measures to quantify reproducibility. Inf. Process. Manag. 60(3), 103332 (2023)","journal-title":"Inf. Process. Manag."},{"key":"16_CR22","doi-asserted-by":"crossref","unstructured":"Pradeep, R., Nogueira, R.F., Lin, J.: The expando-mono-duo design pattern for text ranking with pretrained sequence-to-sequence models. CoRR abs\/2101.05667 (2021)","DOI":"10.18653\/v1\/2020.findings-emnlp.63"},{"key":"16_CR23","doi-asserted-by":"crossref","unstructured":"Robertson, S.E., Walker, S., Jones, S., Hancock-Beaulieu, M., Gatford, M.: Okapi at TREC-3. In: TREC. NIST Special Publication, vol. 500-225, pp. 109\u2013126. National Institute of Standards and Technology (NIST) (1994)","DOI":"10.6028\/NIST.SP.500-225.routing-city"},{"key":"16_CR24","unstructured":"Saez, G.G.: Continuous evaluation framework for information retrieval systems. Theses, Universit\u00e9 Grenoble Alpes [2020-....] (2023). https:\/\/theses.hal.science\/tel-04547265"},{"key":"16_CR25","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"91","DOI":"10.1007\/978-3-030-85251-1_8","volume-title":"Experimental IR Meets Multilinguality, Multimodality, and Interaction","author":"GN Gonz\u00e1lez-S\u00e1ez","year":"2021","unstructured":"Gonz\u00e1lez-S\u00e1ez, G.N., Mulhem, P., Goeuriot, L.: Towards the evaluation of information retrieval systems on evolving datasets with pivot systems. In: Candan, K.S., et al. (eds.) CLEF 2021. LNCS, vol. 12880, pp. 91\u2013102. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-85251-1_8"},{"key":"16_CR26","doi-asserted-by":"crossref","unstructured":"Soboroff, I.: Dynamic test collections: measuring search effectiveness on the live web. In: SIGIR, pp. 276\u2013283. ACM (2006)","DOI":"10.1145\/1148170.1148220"},{"key":"16_CR27","doi-asserted-by":"crossref","unstructured":"Tikhonov, A., Bogatyy, I., Burangulov, P., Ostroumova, L., Koshelev, V., Gusev, G.: Studying page life patterns in dynamical web. In: SIGIR, pp. 905\u2013908. ACM (2013)","DOI":"10.1145\/2484028.2484185"},{"issue":"5","key":"16_CR28","doi-asserted-by":"publisher","first-page":"445","DOI":"10.1007\/s10791-015-9266-y","volume":"18","author":"A Tonon","year":"2015","unstructured":"Tonon, A., Demartini, G., Cudr\u00e9-Mauroux, P.: Pooling-based continuous evaluation of information retrieval systems. Inf. Retr. J. 18(5), 445\u2013472 (2015)","journal-title":"Inf. Retr. J."},{"key":"16_CR29","unstructured":"Wang, L., et al.: Text embeddings by weakly-supervised contrastive pre-training. CoRR abs\/2212.03533 (2022)"}],"container-title":["Lecture Notes in Computer Science","Experimental IR Meets Multilinguality, Multimodality, and Interaction"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-71736-9_16","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,19]],"date-time":"2024-10-19T14:07:06Z","timestamp":1729346826000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-71736-9_16"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"ISBN":["9783031717352","9783031717369"],"references-count":29,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-71736-9_16","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2024]]},"assertion":[{"value":"14 September 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors have no competing interests to declare that are relevant to the content of this article.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"CLEF","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference of the Cross-Language Evaluation Forum for European Languages","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Grenoble","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"France","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"9 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"12 September 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"15","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"clef2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/clef2024.clef-initiative.eu\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}