{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:18:24Z","timestamp":1784179104733,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":54,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,7,13]]},"DOI":"10.1145\/3726302.3730308","type":"proceedings-article","created":{"date-parts":[[2025,7,14]],"date-time":"2025-07-14T14:55:26Z","timestamp":1752504926000},"page":"3387-3397","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["Variations in Relevance Judgments and the Shelf Life of Test Collections"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5446-8328","authenticated-orcid":false,"given":"Andrew","family":"Parry","sequence":"first","affiliation":[{"name":"University of Glasgow, Glasgow, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1003-981X","authenticated-orcid":false,"given":"Maik","family":"Fr\u00f6be","sequence":"additional","affiliation":[{"name":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9578-7157","authenticated-orcid":false,"given":"Harrisen","family":"Scells","sequence":"additional","affiliation":[{"name":"University of T\u00fcbingen, Germany, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6032-909X","authenticated-orcid":false,"given":"Ferdinand","family":"Schlatt","sequence":"additional","affiliation":[{"name":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5070-2049","authenticated-orcid":false,"given":"Guglielmo","family":"Faggioli","sequence":"additional","affiliation":[{"name":"University of Padua, Padua, Italy"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2259-0462","authenticated-orcid":false,"given":"Saber","family":"Zerhoudi","sequence":"additional","affiliation":[{"name":"Universit\u00e4t Passau, Passau, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8914-2659","authenticated-orcid":false,"given":"Sean","family":"MacAvaney","sequence":"additional","affiliation":[{"name":"University of Glasgow, Glasgow, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0051-1535","authenticated-orcid":false,"given":"Eugene","family":"Yang","sequence":"additional","affiliation":[{"name":"Johns Hopkins University, Baltimore, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,7,13]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210176"},{"key":"e_1_3_2_1_2_1","unstructured":"Xavier Amatriain and Justin Basilico. 2017. Netflix recommendations: Beyond the 5 stars (part 1). https:\/\/netflixtechblog.com\/netflix-recommendations-beyondthe-5-stars-part-1-55838468f429"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646031"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1016\/0306-4573(92)90031-T"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646033"},{"key":"e_1_3_2_1_6_1","volume-title":"Technical Report PB169574. Association","author":"Cleverdon C.","year":"1966","unstructured":"C. Cleverdon, J. Mills, and M. Keen. 1966. Factors Determining the Performance of Indexing Systems. Volume I. Design. Part 2. Appendices. Technical Report PB169574. Association of Special Libraries and Information Bureau, Cranfield (England). https:\/\/ntrl.ntis.gov\/NTRL\/dashboard\/searchResults\/titleDetail\/PB169574.xhtml Num Pages: 261."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.6028\/NIST.SP.1266.deep-overview"},{"key":"e_1_3_2_1_8_1","volume-title":"Overview of the TREC 2019 Deep Learning Track. In 28th International Text Retrieval Conference, TREC 2019","author":"Craswell Nick","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, Daniel Campos, and Ellen M. Voorhees. 2019. Overview of the TREC 2019 Deep Learning Track. In 28th International Text Retrieval Conference, TREC 2019, Gaithersburg, Maryland, USA (NIST Special Publication), Ellen M. Voorhees and Angela Ellis (Eds.). National Institute of Standards and Technology (NIST)."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463249"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1108\/eb026436"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531857"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463098"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-28244-7_20"},{"key":"e_1_3_2_1_14_1","volume-title":"Overview of the Fourth Text REtrieval Conference (TREC- . In Proceedings of The Fourth Text REtrieval Conference, TREC 1995","author":"Harman Donna","year":"1995","unstructured":"Donna Harman. 1995. Overview of the Fourth Text REtrieval Conference (TREC- . In Proceedings of The Fourth Text REtrieval Conference, TREC 1995, Gaithersburg, Maryland, USA, November 1-3, 1995 (NIST Special Publication, Vol. 500-236), Donna K. Harman (Ed.). National Institute of Standards and Technology (NIST). http:\/\/trec.nist.gov\/pubs\/trec4\/overview.ps.gz"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-36415-0_7"},{"key":"e_1_3_2_1_16_1","volume-title":"Distilling the Knowledge in a Neural Network. arXiv preprint arXiv:1503.02531","author":"Hinton Geoffrey","year":"2015","unstructured":"Geoffrey Hinton. 2015. Distilling the Knowledge in a Neural Network. arXiv preprint arXiv:1503.02531 (2015)."},{"key":"e_1_3_2_1_17_1","volume-title":"Improving Efficient Neural Ranking Models with Cross-Architecture Knowledge Distillation. CoRR abs\/2010.02666","author":"Hofst\u00e4tter Sebastian","year":"2020","unstructured":"Sebastian Hofst\u00e4tter, Sophia Althammer, Michael Schr\u00f6der, Mete Sertkan, and Allan Hanbury. 2020. Improving Efficient Neural Ranking Models with Cross-Architecture Knowledge Distillation. CoRR abs\/2010.02666 (2020). arXiv:2010.02666 https:\/\/arxiv.org\/abs\/2010.02666"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1111\/lnc3.12432"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_1_20_1","unstructured":"Jaap Kamps and David Rau. [n. d.]. University of Amsterdam at TREC 2021: Deep Learning Track. https:\/\/api.semanticscholar.org\/CorpusID:266088115"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.550"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401075"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1016\/0020-0271(68)90029-6"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3687273.3687287"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.5555\/262192.262203"},{"key":"e_1_3_2_1_26_1","unstructured":"Tri Nguyen Mir Rosenberg Xia Song Jianfeng Gao Saurabh Tiwary Rangan Majumder and Li Deng. 2016. MS MARCO: A Human Generated MAchine Reading COmprehension Dataset. In Proceedings of the Workshop on Cognitive Computation: Integrating neural and symbolic approaches 2016 co-located with the 30th Annual Conference on Neural Information Processing Systems (NIPS 2016) Barcelona Spain December 9 2016 (CEUR Workshop Proceedings Vol. 1773) Tarek Richard Besold Antoine Bordes Artur S. d'Avila Garcez and Greg Wayne (Eds.). CEUR-WS.org."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.63"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3536321"},{"key":"e_1_3_2_1_29_1","unstructured":"Andrew Parry Sean MacAvaney and Debasis Ganguly. 2024. Top-Down Partitioning for Efficient List-Wise Ranking. arXiv:2405.14589 [cs.IR] https:\/\/arxiv.org\/abs\/2405.14589"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2348283.2348429"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-99736-6_44"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2312.02724"},{"key":"e_1_3_2_1_33_1","volume-title":"Schultz","author":"Rees Alan M.","year":"1967","unstructured":"Alan M. Rees and Douglas G. Schultz. 1967. A Field Experimental Approach to the Study of Relevance Assessments in Relation to Document Searching. Final Report to the National Science Foundation. Volume I. Technical Report. Clearinghouse for Federal Scientific and Technical Information, Springfield, Va."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1097-4571(198807)39:4<235::AID-ASI3>"},{"key":"e_1_3_2_1_35_1","volume-title":"Proceedings of The Fourth Text REtrieval Conference, TREC 1995","author":"Robertson Stephen E.","year":"1995","unstructured":"Stephen E. Robertson, Steve Walker, Micheline Hancock-Beaulieu, Mike Gatford, and A. Payne. 1995. Okapi at TREC-4. In Proceedings of The Fourth Text REtrieval Conference, TREC 1995, Gaithersburg, Maryland, USA, November 1-3, 1995 (NIST Special Publication, Vol. 500-236), Donna K. Harman (Ed.). National Institute of Standards and Technology (NIST). http:\/\/trec.nist.gov\/pubs\/trec4\/papers\/city.ps. gz"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/1458082.1458159"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1016\/0306-"},{"key":"e_1_3_2_1_39_1","volume-title":"Set-Encoder: Permutation-Invariant Inter-Passage Attention for Listwise Passage Re-Ranking with Cross-Encoders. arXiv preprint arXiv:2404.06912","author":"Schlatt Ferdinand","year":"2024","unstructured":"Ferdinand Schlatt, Maik Fr\u00f6be, Harrisen Scells, Shengyao Zhuang, Bevan Koopman, Guido Zuccon, Benno Stein, Martin Potthast, and Matthias Hagen. 2024. Set-Encoder: Permutation-Invariant Inter-Passage Attention for Listwise Passage Re-Ranking with Cross-Encoders. arXiv preprint arXiv:2404.06912 (2024)."},{"key":"e_1_3_2_1_40_1","unstructured":"Ferdinand Schlatt Maik Fr\u00f6be Harrisen Scells Shengyao Zhuang Bevan Koopman Guido Zuccon Benno Stein Martin Potthast and Matthias Hagen. 2024. A Systematic Investigation of Distilling Large Language Models into Cross- Encoders for Passage Re-ranking. arXiv:2405.07920 [cs.IR] https:\/\/arxiv.org\/abs\/ 2405.07920"},{"key":"e_1_3_2_1_41_1","volume-title":"LexMAE: Lexicon-Bottlenecked Pretraining for Large-Scale Retrieval. ArXiv abs\/2208.14754","author":"Shen Tao","year":"2022","unstructured":"Tao Shen, Xiubo Geng, Chongyang Tao, Can Xu, Xiaolong Huang, Binxing Jiao, Linjun Yang, and Daxin Jiang. 2022. LexMAE: Lexicon-Bottlenecked Pretraining for Large-Scale Retrieval. ArXiv abs\/2208.14754 (2022). https: \/\/api.semanticscholar.org\/CorpusID:251953412"},{"key":"e_1_3_2_1_42_1","unstructured":"Ian Soboroff. 2024. Don't Use LLMs to Make Relevance Judgments. arXiv:2409.15133 [cs.IR] https:\/\/arxiv.org\/abs\/2409.15133"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.923"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3653712"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.5090160204"},{"key":"e_1_3_2_1_46_1","volume-title":"Variations in Relevance Judgments and the Measurement of Retrieval Effectiveness. 36 No. 5 (2000-01-01","author":"Voorhees Ellen","year":"2000","unstructured":"Ellen Voorhees. 2000. Variations in Relevance Judgments and the Measurement of Retrieval Effectiveness. 36 No. 5 (2000-01-01 2000)."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291017"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-22948-1_2"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531728"},{"key":"e_1_3_2_1_50_1","first-page":"3","volume-title":"Overview of the Sixth Text REtrieval Conference (TREC-6). Inf. Process. Manag. 36","author":"Ellen","year":"1997","unstructured":"Ellen M. Voorhees and Donna K. Harman. 1997. Overview of the Sixth Text REtrieval Conference (TREC-6). Inf. Process. Manag. 36 (1997), 3-35. https:\/\/api.semanticscholar.org\/CorpusID:9854240"},{"key":"e_1_3_2_1_51_1","volume-title":"SimLM: Pre-training with Representation Bottleneck for Dense Passage Retrieval. In Annual Meeting of the Association for Computational Linguistics. https:\/\/api.semanticscholar.org\/CorpusID:250311114","author":"Wang Liang","year":"2022","unstructured":"Liang Wang, Nan Yang, Xiaolong Huang, Binxing Jiao, Linjun Yang, Daxin Jiang, Rangan Majumder, and FuruWei. 2022. SimLM: Pre-training with Representation Bottleneck for Dense Passage Retrieval. In Annual Meeting of the Association for Computational Linguistics. https:\/\/api.semanticscholar.org\/CorpusID:250311114"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.125"},{"key":"e_1_3_2_1_53_1","first-page":"4738","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence","volume":"37","author":"Ma Guangyuan","year":"2023","unstructured":"XingWu, Guangyuan Ma, Meng Lin, Zijia Lin, ZhongyuanWang, and Songlin Hu. 2023. Contextual masked auto-encoder for dense passage retrieval. In Proceedings of the AAAI Conference on Artificial Intelligence, Vol. 37. 4738-4746."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR '25: The 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Padua Italy","acronym":"SIGIR '25","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3726302.3730308","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T18:34:50Z","timestamp":1755887690000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3726302.3730308"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,13]]},"references-count":54,"alternative-id":["10.1145\/3726302.3730308","10.1145\/3726302"],"URL":"https:\/\/doi.org\/10.1145\/3726302.3730308","relation":{},"subject":[],"published":{"date-parts":[[2025,7,13]]},"assertion":[{"value":"2025-07-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}