{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,19]],"date-time":"2025-08-19T10:23:09Z","timestamp":1755598989860,"version":"3.40.3"},"publisher-location":"Cham","reference-count":38,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783031136429"},{"type":"electronic","value":"9783031136436"}],"license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022]]},"DOI":"10.1007\/978-3-031-13643-6_4","type":"book-chapter","created":{"date-parts":[[2022,8,24]],"date-time":"2022-08-24T15:03:19Z","timestamp":1661353399000},"page":"48-61","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Noise-Reduction for\u00a0Automatically Transferred Relevance Judgments"],"prefix":"10.1007","author":[{"given":"Maik","family":"Fr\u00f6be","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Christopher","family":"Akiki","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Martin","family":"Potthast","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Matthias","family":"Hagen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2022,8,25]]},"reference":[{"key":"4_CR1","unstructured":"Arabzadeh, N., Vtyurina, A., Yan, X., Clarke, C.: Shallow pooling for sparse labels. CoRR abs\/2109.00062 (2021)"},{"key":"4_CR2","unstructured":"Bevendorff, J., Potthast, M., Stein, B.: FastWARC: optimizing large-scale web archive analytics. In: Proceeding of OSSYM 2021. OSF (2021)"},{"key":"4_CR3","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"820","DOI":"10.1007\/978-3-319-76941-7_83","volume-title":"Advances in Information Retrieval","author":"J Bevendorff","year":"2018","unstructured":"Bevendorff, J., Stein, B., Hagen, M., Potthast, M.: Elastic chatnoir: search engine for the clueweb and the common crawl. In: Pasi, G., Piwowarski, B., Azzopardi, L., Hanbury, A. (eds.) ECIR 2018. LNCS, vol. 10772, pp. 820\u2013824. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-319-76941-7_83"},{"key":"4_CR4","doi-asserted-by":"crossref","unstructured":"Cen, R., Liu, Y., Zhang, M., Zhou, B., Ru, L., Ma, S.: Exploring relevance for clicks. In: Proceeding of CIKM 2009, pp. 1847\u20131850. ACM (2009)","DOI":"10.1145\/1645953.1646246"},{"key":"4_CR5","unstructured":"Cho, J., Garcia-Molina, H.: The evolution of the web and implications for an incremental crawler. In: Proceeding of VLDB 2000, pp. 200\u2013209 (2000)"},{"key":"4_CR6","doi-asserted-by":"crossref","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D.: Overview of the TREC 2020 deep learning track. In: Proceeding of TREC 2020. NIST (2020)","DOI":"10.6028\/NIST.SP.1266.deep-overview"},{"key":"4_CR7","doi-asserted-by":"crossref","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D., Lin, J.: MS MARCO: benchmarking ranking models in the large-data regime. In: Proceeding of SIGIR 2021, pp. 1566\u20131576. ACM (2021)","DOI":"10.1145\/3404835.3462804"},{"key":"4_CR8","doi-asserted-by":"crossref","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D., Voorhees, E.: Overview of the TREC 2019 deep learning track. In: Proceeding of TREC 2019. NIST (2019)","DOI":"10.6028\/NIST.SP.1266.deep-overview"},{"key":"4_CR9","doi-asserted-by":"crossref","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D.: Overview of the TREC 2021 deep learning track. In: Voorhees, E.M., Ellis, A. (eds.) Notebook. NIST (2021)","DOI":"10.6028\/NIST.SP.500-335.deep-overview"},{"key":"4_CR10","doi-asserted-by":"crossref","unstructured":"Dai, Z., Callan, J.: Context-aware document term weighting for ad-hoc search. In: Proceeding of WWW 2020, pp. 1897\u20131907. ACM (2020)","DOI":"10.1145\/3366423.3380258"},{"key":"4_CR11","doi-asserted-by":"crossref","unstructured":"Feng, L., Shu, S., Lin, Z., Lv, F., Li, L., An, B.: Can cross entropy loss be robust to label noise? In: Proceeding of IJCAI 2020, pp. 2206\u20132212. IJCAI (2020)","DOI":"10.24963\/ijcai.2020\/305"},{"key":"4_CR12","doi-asserted-by":"crossref","unstructured":"Fetterly, D., Manasse, M., Najork, M.: On the evolution of clusters of near-duplicate web pages. In: Proceeding of LA-WEB 2003, pp. 37\u201345 (2003)","DOI":"10.1109\/LAWEB.2003.1250280"},{"key":"4_CR13","doi-asserted-by":"crossref","unstructured":"Fetterly, D., Manasse, M., Najork, M., Wiener, J.: A large-scale study of the evolution of web pages. In: Proceeding of WWW 2003, pp. 669\u2013678 (2003)","DOI":"10.1145\/775152.775246"},{"issue":"5","key":"4_CR14","doi-asserted-by":"publisher","first-page":"845","DOI":"10.1109\/TNNLS.2013.2292894","volume":"25","author":"B Fr\u00e9nay","year":"2014","unstructured":"Fr\u00e9nay, B., Verleysen, M.: Classification in the presence of label noise: a survey. IEEE Trans. Neural Netw. Learn. Syst. 25(5), 845\u2013869 (2014)","journal-title":"IEEE Trans. Neural Netw. Learn. Syst."},{"key":"4_CR15","doi-asserted-by":"crossref","unstructured":"Fr\u00f6be, M., et al.: CopyCat: near-duplicates within and between the clueweb and the common crawl. In: Proceeding of SIGIR 2021, pp. 2398\u20132404. ACM (2021)","DOI":"10.1145\/3404835.3463246"},{"key":"4_CR16","unstructured":"Gao, L., Dai, Z., Fan, Z., Callan, J.: Complementing lexical retrieval with semantic residual embedding. CoRR abs\/2004.13969 (2020)"},{"key":"4_CR17","doi-asserted-by":"crossref","unstructured":"Kaszkiel, M., Zobel, J.: Passage retrieval revisited. In: Proceeding of SIGIR 1997, pp. 178\u2013185. ACM (1997)","DOI":"10.1145\/278459.258561"},{"key":"4_CR18","doi-asserted-by":"crossref","unstructured":"Lin, J., Ma, X., Lin, S., Yang, J., Pradeep, R., Nogueira, R.: Pyserini: a python toolkit for reproducible information retrieval research with sparse and dense representations. In: Proceeding of SIGIR 2021, pp. 2356\u20132362. ACM (2021)","DOI":"10.1145\/3404835.3463238"},{"key":"4_CR19","doi-asserted-by":"publisher","unstructured":"MacAvaney, S., Macdonald, C., Ounis, I.: Reproducing personalised session search over the AOL query log. In: Proceeding of ECIR 2022 (2022). https:\/\/doi.org\/10.1007\/978-3-030-99736-6_42","DOI":"10.1007\/978-3-030-99736-6_42"},{"key":"4_CR20","doi-asserted-by":"crossref","unstructured":"MacAvaney, S., Yates, A., Feldman, S., Downey, D., Cohan, A., Goharian, N.: Simplified data wrangling with ir_datasets. In: Proceeding of SIGIR 2021, pp. 2429\u20132436. ACM (2021)","DOI":"10.1145\/3404835.3463254"},{"key":"4_CR21","doi-asserted-by":"crossref","unstructured":"Macdonald, C., Tonellotto, N., MacAvaney, S., Ounis, I.: PyTerrier: declarative experimentation in python from BM25 to dense retrieval. In: Proceeding of CIKM 2021, pp. 4526\u20134533. ACM (2021)","DOI":"10.1145\/3459637.3482013"},{"key":"4_CR22","doi-asserted-by":"crossref","unstructured":"Mokrii, I., Boytsov, L., Braslavski, P.: A systematic evaluation of transfer learning and pseudo-labeling with BERT-based ranking models. In: Proceeding of SIGIR 2021, pp. 2081\u20132085. ACM (2021)","DOI":"10.1145\/3404835.3463093"},{"key":"4_CR23","unstructured":"Nguyen, T., Rosenberg, M., Song, X., Gao, J., Tiwary, S., Majumder, R., Deng, L.: MS MARCO: a human generated machine reading comprehension dataset. In: Proceeding of CoCo@N(eur)IPS 2016. CEUR, vol. 1773. CEUR-WS.org (2016)"},{"key":"4_CR24","doi-asserted-by":"crossref","unstructured":"Nogueira, R., Jiang, Z., Pradeep, R., Lin, J.: Document ranking with a pretrained sequence-to-sequence model. In: Findings of EMNLP 2020, pp. 708\u2013718. ACL (2020)","DOI":"10.18653\/v1\/2020.findings-emnlp.63"},{"key":"4_CR25","unstructured":"Nogueira, R., Yang, W., Cho, K., Lin, J.: Multi-stage document ranking with BERT, pp. 1\u201313. CoRR abs\/1910.14424 (2019)"},{"key":"4_CR26","doi-asserted-by":"crossref","unstructured":"Ntoulas, A., Cho, J., Olston, C.: What\u2019s new on the web? the evolution of the web from a search engine perspective. In: Proceeding of WWW 2004, pp. 1\u201312. ACM (2004)","DOI":"10.1145\/988672.988674"},{"key":"4_CR27","doi-asserted-by":"crossref","unstructured":"Olston, C., Pandey, S.: Recrawl scheduling based on information longevity. In: Proceeding of WWW 2008, pp. 437\u2013446. ACM (2008)","DOI":"10.1145\/1367497.1367557"},{"key":"4_CR28","unstructured":"Pradeep, R., Nogueira, R., Lin, J.: The expando-mono-duo design pattern for text ranking with pretrained sequence-to-sequence models, pp. 1\u201323. CoRR abs\/2101.05667 (2021)"},{"key":"4_CR29","doi-asserted-by":"crossref","unstructured":"Qu, Y., et al.: Rocketqa: An optimized training approach to dense passage retrieval for open-domain question answering. In: Proceeding of NAACL 2021, pp. 5835\u20135847 (2021)","DOI":"10.18653\/v1\/2021.naacl-main.466"},{"key":"4_CR30","unstructured":"Raffel, C., et al.: Exploring the limits of transfer learning with a unified text-to-text transformer. J. Mach. Learn. Res. 21, 140:1\u2013140:67 (2020)"},{"key":"4_CR31","doi-asserted-by":"crossref","unstructured":"Rudra, K., Anand, A.: Distant supervision in bert-based adhoc document retrieval. In: Proceeding of CIKM 2020, pp. 2197\u20132200. ACM (2020)","DOI":"10.1145\/3340531.3412124"},{"key":"4_CR32","doi-asserted-by":"crossref","unstructured":"Singla, A., White, R.: Sampling high-quality clicks from noisy click data. In: Proceeding of WWW 2010, pp. 1187\u20131188. ACM (2010)","DOI":"10.1145\/1772690.1772867"},{"key":"4_CR33","doi-asserted-by":"crossref","unstructured":"Voorhees, E., Craswell, N., Lin, J.: Too many relevants: whither cranfield test collections? In: Proceeding of SIGIR 2022. ACM (2022)","DOI":"10.1145\/3477495.3531728"},{"key":"4_CR34","doi-asserted-by":"publisher","first-page":"6830","DOI":"10.1109\/ACCESS.2022.3142096","volume":"10","author":"X Wu","year":"2022","unstructured":"Wu, X., Liu, Q., Qin, J., Yu, Y.: PeerRank: robust learning to rank with peer loss over noisy labels. IEEE Access 10, 6830\u20136841 (2022)","journal-title":"IEEE Access"},{"key":"4_CR35","doi-asserted-by":"crossref","unstructured":"Yates, A., Arora, S., Zhang, X., Yang, W., Jose, K., Lin, J.: Capreolus: a toolkit for end-to-end neural ad hoc retrieval. In: Proceeding of WSDM 2020, pp. 861\u2013864. ACM (2020)","DOI":"10.1145\/3336191.3371868"},{"key":"4_CR36","doi-asserted-by":"crossref","unstructured":"Yates, A., Nogueira, R., Lin, J.: Pretrained transformers for text ranking: BERT and beyond. In: Proceeding of SIGIR 2021, pp. 2666\u20132668. ACM (2021)","DOI":"10.18653\/v1\/2021.naacl-tutorials.1"},{"key":"4_CR37","doi-asserted-by":"crossref","unstructured":"Zhan, J., Mao, J., Liu, Y., Guo, J., Zhang, M., Ma, S.: Optimizing dense retrieval model training with hard negatives. In: Proceeding of SIGIR 2021, pp. 1503\u20131512. ACM (2021)","DOI":"10.1145\/3404835.3462880"},{"key":"4_CR38","unstructured":"Zhan, J., Mao, J., Liu, Y., Zhang, M., Ma, S.: Repbert: contextualized text embeddings for first-stage retrieval. CoRR abs\/2006.15498 (2020)"}],"container-title":["Lecture Notes in Computer Science","Experimental IR Meets Multilinguality, Multimodality, and Interaction"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-13643-6_4","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,2]],"date-time":"2024-10-02T13:21:17Z","timestamp":1727875277000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-13643-6_4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"ISBN":["9783031136429","9783031136436"],"references-count":38,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-13643-6_4","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2022]]},"assertion":[{"value":"25 August 2022","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"CLEF","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference of the Cross-Language Evaluation Forum for European Languages","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Bologna","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Italy","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"5 September 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"8 September 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"13","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"clef2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/clef2022.clef-initiative.eu\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"14","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"7","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"50% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"7 best of labs + 14 lab overviews","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}