{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T13:40:08Z","timestamp":1755870008343,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":22,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,7,18]]},"DOI":"10.1145\/3731120.3744602","type":"proceedings-article","created":{"date-parts":[[2025,7,18]],"date-time":"2025-07-18T13:34:06Z","timestamp":1752845646000},"page":"326-335","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Impact of Shallow vs. Deep Relevance Judgments on BERT-based Reranking Models"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0001-9635-0683","authenticated-orcid":false,"given":"Gabriel","family":"Iturra-Bocaz","sequence":"first","affiliation":[{"name":"University of Stavanger, Stavanger, Norway"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-1407-9743","authenticated-orcid":false,"given":"Danny","family":"Vo","sequence":"additional","affiliation":[{"name":"University of Stavanger, Stavanger, Norway"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6328-7131","authenticated-orcid":false,"given":"Petra","family":"Galu\u0161\u010d\u00e1kov\u00e1","sequence":"additional","affiliation":[{"name":"University of Stavanger, Stavanger, Norway"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,7,18]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3624918.3625333"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511808.3557343"},{"volume-title":"Introduction to Information Retrieval","author":"Manning Hinrich Sch\u00fctze","key":"e_1_3_2_1_3_1","unstructured":"Hinrich Sch\u00fctze Christopher D. Manning, Prabhakar Raghavan. 2008. Introduction to Information Retrieval. Cambridge University Press."},{"key":"e_1_3_2_1_4_1","volume-title":"Overview of the TREC 2020 Deep Learning Track. In Text REtrieval Conference (TREC'20)","author":"Craswell Nick","year":"2020","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, and Danie Campos. 2020. Overview of the TREC 2020 Deep Learning Track. In Text REtrieval Conference (TREC'20). NIST."},{"key":"e_1_3_2_1_5_1","volume-title":"Overview of the TREC 2021 Deep Learning Track. In Text REtrieval Conference (TREC'21)","author":"Craswell Nick","year":"2021","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, Daniel Campos, and Jimmy Lin. 2021. Overview of the TREC 2021 Deep Learning Track. In Text REtrieval Conference (TREC'21). NIST."},{"key":"e_1_3_2_1_6_1","volume-title":"Overview of the TREC 2022 Deep Learning Track. In Text REtrieval Conference (TREC'22)","author":"Craswell Nick","year":"2022","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, Daniel Campos, Jimmy Lin, Ellen M. Voorhees, and Ian Soboroff. 2022. Overview of the TREC 2022 Deep Learning Track. In Text REtrieval Conference (TREC'22). NIST."},{"volume-title":"Overview of the TREC 2019 Deep Learning Track. In Text REtrieval Conference (TREC'19)","author":"Craswell Nick","key":"e_1_3_2_1_7_1","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, Daniel Campos, and Ellen M. Voorhees. 2019. Overview of the TREC 2019 Deep Learning Track. In Text REtrieval Conference (TREC'19). NIST."},{"key":"e_1_3_2_1_8_1","volume-title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding.","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. (2019). arXiv:1810.04805 [cs.CL]"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"crossref","unstructured":"Maik Fr\u00f6be Christopher Akiki Martin Potthast and Matthias Hagen. 2022. Noise-Reduction for Automatically Transferred Relevance Judgments. In Experimental IR Meets Multilinguality Multimodality and Interaction (CLEF'22). 48-61.","DOI":"10.1007\/978-3-031-13643-6_4"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539618.3591921"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463238"},{"key":"e_1_3_2_1_12_1","first-page":"35","volume-title":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '17)","author":"Lu Xiaolu","unstructured":"Xiaolu Lu, Alistair Moffat, and J. Shane Culpepper. 2017. Can Deep Effectiveness Metrics Be Evaluated Using Shallow Judgment Pools?. In Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '17). 35-44."},{"key":"e_1_3_2_1_13_1","unstructured":"Rodrigo Nogueira and Kyunghyun Cho. 2019. Passage Re-ranking with BERT. (2019). arXiv:1901.04085 [cs.IR]"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"crossref","unstructured":"Rodrigo Nogueira Zhiying Jiang Ronak Pradeep and Jimmy Lin. 2020. Document Ranking with a Pretrained Sequence-to-Sequence Model. In Findings of the Association for Computational Linguistics (EMNLP'20). 708-718.","DOI":"10.18653\/v1\/2020.findings-emnlp.63"},{"key":"e_1_3_2_1_15_1","unstructured":"Rodrigo Nogueira Wei Yang Kyunghyun Cho and Jimmy Lin. 2019. Multi-Stage Document Ranking with BERT. (2019). arXiv:11910.14424 [cs.IR]"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.466"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531728"},{"key":"e_1_3_2_1_18_1","volume-title":"Approximate Nearest Neighbor Negative Contrastive Learning for Dense Text Retrieval. In International Conference on Learning Representations (ICLR'21)","author":"Xiong Lee","year":"2021","unstructured":"Lee Xiong, Chenyan Xiong, Ye Li, Kwok-Fung Tang, Jialin Liu, Paul N. Bennett, Junaid Ahmed, and Arnold Overwijk. 2021. Approximate Nearest Neighbor Negative Contrastive Learning for Dense Text Retrieval. In International Conference on Learning Representations (ICLR'21)."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3437963.3441667"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1572066"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.sustainlp-1.14"},{"volume-title":"How reliable are the results of large-scale information retrieval experiments?","author":"Zobel Justin","key":"e_1_3_2_1_22_1","unstructured":"Justin Zobel. 1998. How reliable are the results of large-scale information retrieval experiments?. In Association for Computing Machinery (SIGIR '98). 307-314."}],"event":{"name":"ICTIR '25: International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Padua Italy","acronym":"ICTIR '25"},"container-title":["Proceedings of the 2025 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval (ICTIR)"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3731120.3744602","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T13:18:34Z","timestamp":1755868714000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3731120.3744602"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,18]]},"references-count":22,"alternative-id":["10.1145\/3731120.3744602","10.1145\/3731120"],"URL":"https:\/\/doi.org\/10.1145\/3731120.3744602","relation":{},"subject":[],"published":{"date-parts":[[2025,7,18]]},"assertion":[{"value":"2025-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}