{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,23]],"date-time":"2026-03-23T23:44:41Z","timestamp":1774309481674,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":49,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,8,2]],"date-time":"2024-08-02T00:00:00Z","timestamp":1722556800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"German Research Foundation","award":["407518790"],"award-info":[{"award-number":["407518790"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,8,2]]},"DOI":"10.1145\/3664190.3672530","type":"proceedings-article","created":{"date-parts":[[2024,8,5]],"date-time":"2024-08-05T12:39:41Z","timestamp":1722861581000},"page":"3-13","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["Evaluation of Temporal Change in IR Test Collections"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9392-8646","authenticated-orcid":false,"given":"J\u00fcri","family":"Keller","sequence":"first","affiliation":[{"name":"TH K\u00f6ln, Cologne, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1765-2449","authenticated-orcid":false,"given":"Timo","family":"Breuer","sequence":"additional","affiliation":[{"name":"TH K\u00f6ln, Cologne, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8817-4632","authenticated-orcid":false,"given":"Philipp","family":"Schaer","sequence":"additional","affiliation":[{"name":"TH K\u00f6ln, Cologne, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,8,5]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1498759.1498837"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1007\/978--3-031--28241--6_58"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","unstructured":"Rabab Alkhalifa Iman Munire Bilal Hsuvas Borkakoty Jos\u00e9 Camacho-Collados Romain Deveaud Alaa El-Ebshihy Luis Espinosa Anke Gabriela Gonz\u00e1lez S\u00e1ez Petra Galusc\u00e1kov\u00e1 Lorraine Goeuriot Elena Kochkina Maria Liakata Daniel Loureiro Philippe Mulhem Florina Piroi Martin Popel Christophe Servan Harish Tayyar Madabushi and Arkaitz Zubiaga. 2023. Overview of the CLEF-2023 LongEval Lab on Longitudinal Evaluation of Model Performance. In Experimental IR Meets Multilinguality Multimodality and Interaction - 14th International Conference of the CLEF Association CLEF 2023 Thessaloniki Greece September 18--21 2023 Proceedings (Lecture Notes in Computer Science Vol. 14163) Avi Arampatzis Evangelos Kanoulas Theodora Tsikrika Stefanos Vrochidis Anastasia Giachanou Dan Li Mohammad Aliannejadi Michalis Vlachos Guglielmo Faggioli and Nicola Ferro (Eds.). Springer 440--458. https:\/\/doi.org\/10.1007\/978--3-031--42448--9_28","DOI":"10.1007\/978--3-031--42448--9_28"},{"key":"e_1_3_2_1_4_1","unstructured":"Giambattista Amati. 2003. Probability Models for Information Retrieval Based on Divergence from Randomness. http:\/\/theses.gla.ac.uk\/1570\/"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1007\/11735106_3"},{"key":"e_1_3_2_1_6_1","volume-title":"Proceedings of the Second International Workshop on Web Dynamics, WebDyn@WWW 2002, Honululu, HW, USA, May 7, 2002 (CEUR Workshop Proceedings","volume":"77","author":"Bar-Ilan Judit","year":"2002","unstructured":"Judit Bar-Ilan. 2002. Criteria for Evaluating Information Retrieval Systems in Highly Dynamic Environments. In Proceedings of the Second International Workshop on Web Dynamics, WebDyn@WWW 2002, Honululu, HW, USA, May 7, 2002 (CEUR Workshop Proceedings, Vol. 702), Mark Levene and Alexandra Poulovassilis (Eds.). CEUR-WS.org, 70--77."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511808.3557207"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401036"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1007\/978--3-030--72240--1_51"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/1008992.1009000"},{"key":"e_1_3_2_1_11_1","volume-title":"Cormack","author":"B\u00fcttcher Stefan","year":"2010","unstructured":"Stefan B\u00fcttcher, Charles L. A. Clarke, and Gordon V. Cormack. 2010. Information Retrieval - Implementing and Evaluating Search Engines. MIT Press. http:\/\/mitpress.mit.edu\/books\/information-retrieval"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/1526709.1526711"},{"key":"e_1_3_2_1_13_1","volume-title":"Novelty and Diversity in Information Retrieval Evaluation. In Annual International ACM SIGIR Conference on Research and Development in Information Retrieval.","author":"Clarke Charles L. A.","year":"2008","unstructured":"Charles L. A. Clarke, Maheedhar Kolla, Gordon V. Cormack, Olga Vechtomova, Azin Ashkan, Stefan B\u00fcttcher, and Ian MacKinnon. 2008. Novelty and Diversity in Information Retrieval Evaluation. In Annual International ACM SIGIR Conference on Research and Development in Information Retrieval."},{"key":"e_1_3_2_1_14_1","volume-title":"Readings in Information Retrieval","author":"Cleverdon Cyril","unstructured":"Cyril Cleverdon. 1997. The Cranfield Tests on Index Language Devices. In Readings in Information Retrieval. Morgan Kaufmann Publishers Inc., San Francisco, CA, USA, 47--59."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1572114"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/1341531.1341545"},{"key":"e_1_3_2_1_17_1","unstructured":"W. Bruce Croft Donald Metzler and Trevor Strohman. 2009. Search Engines - Information Retrieval in Practice. Pearson Education. http:\/\/www.search-engines-book.com\/"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"crossref","unstructured":"Susan Dumais. 2014. Putting Searchers into Search. In Athena Lecture at SIGIR 2014 athena lecture at sigir 2014 ed.). https:\/\/www.microsoft.com\/en-us\/research\/publication\/putting-searchers-search\/","DOI":"10.1145\/2600428.2617557"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/1718487.1718489"},{"key":"e_1_3_2_1_20_1","unstructured":"Gabriela G. Saez. 2023. Continuous Evaluation Framework for Information Retrieval Systems. Theses. Universit\u00e9 Grenoble Alpes [2020-....]. https:\/\/theses.hal.science\/tel-04547265"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539618.3591921"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1007\/978--3-030--22948--1_21"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1007\/1--4020--3851--8"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1292591.1292592"},{"key":"e_1_3_2_1_26_1","volume-title":"Working Notes of the Conference and Labs of the Evaluation Forum (CLEF","author":"Keller J\u00fcri","year":"2023","unstructured":"J\u00fcri Keller, Timo Breuer, and Philipp Schaer. 2023. Evaluating Temporal Persistence Using Replicability Measures. In Working Notes of the Conference and Labs of the Evaluation Forum (CLEF 2023), Thessaloniki, Greece, September 18th to 21st, 2023 (CEUR Workshop Proceedings, Vol. 3497), Mohammad Aliannejadi, Guglielmo Faggioli, Nicola Ferro, and Michalis Vlachos (Eds.). CEUR-WS.org, 2441--2457. https:\/\/ceur-ws.org\/Vol-3497\/paper-196.pdf"},{"key":"e_1_3_2_1_27_1","volume-title":"Proceedings of the Fifteenth International Conference of the CLEF Association (CLEF 2024)","author":"Keller J\u00fcri","year":"2024","unstructured":"J\u00fcri Keller, Timo Breuer, and Philipp Schaer. 2024. Cross-Lingual Candidate Retrieval and Re-ranking for Biomedical Entity Linking. In Experimental IR Meets Multilinguality, Multimodality, and Interaction. Proceedings of the Fifteenth International Conference of the CLEF Association (CLEF 2024) (Lecture Notes in Computer Science), Lorraine Goeuriot, Philippe Mulhem, Georges Qu\u00e9not, Didier Schwab, Giorgio Maria Di Nunzio, Laure Soulier, Petra Galuscakova, Alba Garc\u00eda Seco de Herrera, Guglielmo Faggioli, and Nicola Ferro (Eds.). Springer."},{"key":"e_1_3_2_1_28_1","unstructured":"Maurice G. Kendall. 1948. Rank correlation methods.Griffin."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401075"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/1935826.1935862"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3409256.3409829"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2023.103332"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511809071"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3121050.3121100"},{"key":"e_1_3_2_1_35_1","unstructured":"James Martin. 1983. Managing the data-base environment. https:\/\/api.semanticscholar.org\/CorpusID:109367417"},{"key":"e_1_3_2_1_36_1","unstructured":"Rodrigo Nogueira and Jimmy Lin. 2019. From Doc2query to docTTTTTquery. https:\/\/cs.uwaterloo.ca\/ jimmylin\/publications\/Nogueira_Lin_2019_docTTTTTquery-v2.pdf"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.63"},{"key":"e_1_3_2_1_38_1","unstructured":"Ronak Pradeep Rodrigo Nogueira and Jimmy Lin. 2021. The Expando-Mono-Duo Design Pattern for Text Ranking with Pretrained Sequence-to-Sequence Models. arxiv: 2101.05667 [cs] http:\/\/arxiv.org\/abs\/2101.05667"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463242"},{"key":"e_1_3_2_1_40_1","volume-title":"Proceedings of The Third Text REtrieval Conference, TREC 1994","author":"Robertson Stephen E.","year":"1994","unstructured":"Stephen E. Robertson, Steve Walker, Susan Jones, Micheline Hancock-Beaulieu, and Mike Gatford. 1994. Okapi at TREC-3. In Proceedings of The Third Text REtrieval Conference, TREC 1994, Gaithersburg, Maryland, USA, November 2--4, 1994 (NIST Special Publication, Vol. 500--225), Donna K. Harman (Ed.). National Institute of Standards and Technology (NIST), 109--126. http:\/\/trec.nist.gov\/pubs\/trec3\/papers\/city.ps.gz"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.24348\/coria.2021.long_6"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1007\/978--3-030--85251--1_8"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277756"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148220"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080703"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484185"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-015--9266-y"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3451964.3451965"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/1852102.1852106"}],"event":{"name":"ICTIR '24: The 2024 ACM SIGIR International Conference on the Theory of Information Retrieval","location":"Washington DC USA","acronym":"ICTIR '24","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 2024 ACM SIGIR International Conference on Theory of Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3664190.3672530","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3664190.3672530","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T23:58:41Z","timestamp":1755907121000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3664190.3672530"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,8,2]]},"references-count":49,"alternative-id":["10.1145\/3664190.3672530","10.1145\/3664190"],"URL":"https:\/\/doi.org\/10.1145\/3664190.3672530","relation":{},"subject":[],"published":{"date-parts":[[2024,8,2]]},"assertion":[{"value":"2024-08-05","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}