{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:15:01Z","timestamp":1750220101724,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":31,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,7,6]],"date-time":"2022-07-06T00:00:00Z","timestamp":1657065600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,7,6]]},"DOI":"10.1145\/3477495.3532006","type":"proceedings-article","created":{"date-parts":[[2022,7,7]],"date-time":"2022-07-07T15:12:08Z","timestamp":1657206728000},"page":"556-566","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["IR Evaluation and Learning in the Presence of Forbidden Documents"],"prefix":"10.1145","author":[{"given":"David","family":"Carmel","sequence":"first","affiliation":[{"name":"Amazon, Haifa, Israel"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nachshon","family":"Cohen","sequence":"additional","affiliation":[{"name":"Amazon, Haifa, Israel"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Amir","family":"Ingber","sequence":"additional","affiliation":[{"name":"Pinecone Systems, Haifa, Israel"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Elad","family":"Kravi","sequence":"additional","affiliation":[{"name":"Amazon, Haifa, Israel"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,7,7]]},"reference":[{"volume-title":"HuggingFace transformers package. https:\/\/huggingface.co\/transformers\/ model_doc\/distilbert.html. [Online","year":"2021","key":"e_1_3_2_2_1_1","unstructured":"2020. HuggingFace transformers package. https:\/\/huggingface.co\/transformers\/ model_doc\/distilbert.html. [Online ; accessed 25- May - 2021 2020. HuggingFace transformers package. https:\/\/huggingface.co\/transformers\/ model_doc\/distilbert.html. [Online; accessed 25-May-2021"},{"key":"e_1_3_2_2_2_1","volume-title":"The Dark Side of Relevance: The Effect of Non-Relevant Results on Search Behavior. In ACM SIGIR Conference on Human Information Interaction and Retrieval","author":"Abualsaud Mustafa","year":"2022","unstructured":"Mustafa Abualsaud and Mark Smucker . 2022 . The Dark Side of Relevance: The Effect of Non-Relevant Results on Search Behavior. In ACM SIGIR Conference on Human Information Interaction and Retrieval ( Regensburg, Germany) (CHIIR '22). Association for Computing Machinery, 1--11. https:\/\/doi.org\/10.1145\/3498366.3505770 10.1145\/3498366.3505770 Mustafa Abualsaud and Mark Smucker. 2022. The Dark Side of Relevance: The Effect of Non-Relevant Results on Search Behavior. In ACM SIGIR Conference on Human Information Interaction and Retrieval (Regensburg, Germany) (CHIIR '22). Association for Computing Machinery, 1--11. https:\/\/doi.org\/10.1145\/3498366.3505770"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1572031"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277805"},{"volume-title":"Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval","author":"Buckley Chris","key":"e_1_3_2_2_5_1","unstructured":"Chris Buckley and Ellen M. Voorhees . 2004. Retrieval Evaluation with Incomplete Information . In Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval ( Sheffield, United Kingdom) (SIGIR '04). Association for Computing Machinery, 25--32. https:\/\/doi.org\/10.1145\/1008992.1009000 10.1145\/1008992.1009000 Chris Buckley and Ellen M. Voorhees. 2004. Retrieval Evaluation with Incomplete Information. In Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (Sheffield, United Kingdom) (SIGIR '04). Association for Computing Machinery, 25--32. https:\/\/doi.org\/10.1145\/1008992.1009000"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3130348.3130373"},{"key":"e_1_3_2_2_8_1","volume-title":"Overview of the TREC 2010 web track. In Proceedings of the 19th Text REtrieval Conference.","author":"Clarke Charles L","year":"2010","unstructured":"Charles L Clarke , Nick Craswell , and Ian M Soboroff . 2010 . Overview of the TREC 2010 web track. In Proceedings of the 19th Text REtrieval Conference. Charles L Clarke, Nick Craswell, and Ian M Soboroff. 2010. Overview of the TREC 2010 web track. In Proceedings of the 19th Text REtrieval Conference."},{"key":"e_1_3_2_2_9_1","volume-title":"Overview of the TREC 2012 web track. In Proceedings of the 21th Text REtrieval Conference.","author":"Clarke Charles L","year":"2012","unstructured":"Charles L Clarke , Nick Craswell , and Ellen M Voorhees . 2012 . Overview of the TREC 2012 web track. In Proceedings of the 21th Text REtrieval Conference. Charles L Clarke, Nick Craswell, and Ellen M Voorhees. 2012. Overview of the TREC 2012 web track. In Proceedings of the 21th Text REtrieval Conference."},{"key":"e_1_3_2_2_10_1","volume-title":"Overview of the TREC 2013 web track. In Proceedings of the 22th Text REtrieval Conference.","author":"Collins-Thompson Kevyn","year":"2013","unstructured":"Kevyn Collins-Thompson , Paul Bennett , Fernando Diaz , CLA Clarke , and EM Voorhees . 2013 . Overview of the TREC 2013 web track. In Proceedings of the 22th Text REtrieval Conference. Kevyn Collins-Thompson, Paul Bennett, Fernando Diaz, CLA Clarke, and EM Voorhees. 2013. Overview of the TREC 2013 web track. In Proceedings of the 22th Text REtrieval Conference."},{"key":"e_1_3_2_2_11_1","volume-title":"TREC 2014 web track overview. In Proceedings of the 23th Text REtrieval Conference.","author":"Collins-Thompson Kevyn","year":"2014","unstructured":"Kevyn Collins-Thompson , Craig Macdonald , Paul Bennett , Fernando Diaz , and Ellen M Voorhees . 2014 . TREC 2014 web track overview. In Proceedings of the 23th Text REtrieval Conference. Kevyn Collins-Thompson, Craig Macdonald, Paul Bennett, Fernando Diaz, and Ellen M Voorhees. 2014. TREC 2014 web track overview. In Proceedings of the 23th Text REtrieval Conference."},{"key":"e_1_3_2_2_12_1","first-page":"5","article-title":"Efficient and Effective Spam Filtering and Re-Ranking for Large Web","volume":"14","author":"Cormack Gordon V.","year":"2011","unstructured":"Gordon V. Cormack , Mark D. Smucker , and Charles L. Clarke . 2011 . Efficient and Effective Spam Filtering and Re-Ranking for Large Web Datasets. Inf. Retr. , Vol. 14 , 5 (oct 2011), 441--465. https:\/\/doi.org\/10.1007\/s10791-011--9162-z 10.1007\/s10791-011--9162-z Gordon V. Cormack, Mark D. Smucker, and Charles L. Clarke. 2011. Efficient and Effective Spam Filtering and Re-Ranking for Large Web Datasets. Inf. Retr., Vol. 14, 5 (oct 2011), 441--465. https:\/\/doi.org\/10.1007\/s10791-011--9162-z","journal-title":"Datasets. Inf. Retr."},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.5555\/645328.649997"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3412123"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-2016"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646032"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3121050.3121072"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2914737"},{"volume-title":"Learning to Rank for Information Retrieval","author":"Liu Tie-Yan","key":"e_1_3_2_2_20_1","unstructured":"Tie-Yan Liu . 2011. Learning to Rank for Information Retrieval . Springer . I--XVII, 1--285 pages. Tie-Yan Liu. 2011. Learning to Rank for Information Retrieval. Springer. I--XVII, 1--285 pages."},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3459637.3482287"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-45068-6_1"},{"key":"e_1_3_2_2_23_1","volume-title":"Context-Aware Learning to Rank with Self-Attention. ArXiv","author":"Pobrotyn Przemyslaw","year":"2020","unstructured":"Przemyslaw Pobrotyn , Tomasz Bartczak , Mikolaj Synowiec , Radoslaw Bialobrzeski , and Jaroslaw Bojar . 2020. Context-Aware Learning to Rank with Self-Attention. ArXiv , Vol. abs\/ 2005 .10084 ( 2020 ). Przemyslaw Pobrotyn, Tomasz Bartczak, Mikolaj Synowiec, Radoslaw Bialobrzeski, and Jaroslaw Bojar. 2020. Context-Aware Learning to Rank with Self-Attention. ArXiv, Vol. abs\/2005.10084 (2020)."},{"key":"e_1_3_2_2_24_1","volume-title":"TREC 2015 Total Recall Track Overview.. In TREC.","author":"Roegiest Adam","year":"2015","unstructured":"Adam Roegiest , Gordon V Cormack , Charles LA Clarke , and Maura R Grossman . 2015 . TREC 2015 Total Recall Track Overview.. In TREC. Adam Roegiest, Gordon V Cormack, Charles LA Clarke, and Maura R Grossman. 2015. TREC 2015 Total Recall Track Overview.. In TREC."},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148261"},{"key":"e_1_3_2_2_26_1","volume-title":"a distilled version of BERT: smaller, faster, cheaper and lighter. arXiv preprint arXiv:1910.01108","author":"Sanh Victor","year":"2019","unstructured":"Victor Sanh , Lysandre Debut , Julien Chaumond , and Thomas Wolf . 2019. DistilBERT , a distilled version of BERT: smaller, faster, cheaper and lighter. arXiv preprint arXiv:1910.01108 ( 2019 ). Victor Sanh, Lysandre Debut, Julien Chaumond, and Thomas Wolf. 2019. DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter. arXiv preprint arXiv:1910.01108 (2019)."},{"key":"e_1_3_2_2_27_1","volume-title":"Overview of the TREC 2011 web track. In Proceedings of the 20th Text REtrieval Conference.","author":"Soboroff Ian M","year":"2011","unstructured":"Ian M Soboroff , Nick Craswell , Charles L Clarke , and Gordon Cormack . 2011 . Overview of the TREC 2011 web track. In Proceedings of the 20th Text REtrieval Conference. Ian M Soboroff, Nick Craswell, Charles L Clarke, and Gordon Cormack. 2011. Overview of the TREC 2011 web track. In Proceedings of the 20th Text REtrieval Conference."},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3271784"},{"key":"e_1_3_2_2_29_1","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations. Association for Computational Linguistics, Online, 38--45","author":"Wolf Thomas","year":"2020","unstructured":"Thomas Wolf , Lysandre Debut , Victor Sanh , Julien Chaumond , Clement Delangue , Anthony Moi , Pierric Cistac , Tim Rault , R\u00e9mi Louf , Morgan Funtowicz , Joe Davison , Sam Shleifer , Patrick von Platen , Clara Ma , Yacine Jernite , Julien Plu , Canwen Xu , Teven Le Scao , Sylvain Gugger , Mariama Drame , Quentin Lhoest , and Alexander M. Rush . 2020. Transformers: State-of-the-Art Natural Language Processing . In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations. Association for Computational Linguistics, Online, 38--45 . https:\/\/www.aclweb.org\/anthology\/ 2020 .emnlp-demos.6 Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Clement Delangue, Anthony Moi, Pierric Cistac, Tim Rault, R\u00e9mi Louf, Morgan Funtowicz, Joe Davison, Sam Shleifer, Patrick von Platen, Clara Ma, Yacine Jernite, Julien Plu, Canwen Xu, Teven Le Scao, Sylvain Gugger, Mariama Drame, Quentin Lhoest, and Alexander M. Rush. 2020. Transformers: State-of-the-Art Natural Language Processing. In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations. Association for Computational Linguistics, Online, 38--45. https:\/\/www.aclweb.org\/anthology\/2020.emnlp-demos.6"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i5.16572"},{"key":"e_1_3_2_2_31_1","volume-title":"Jeffrey Chan, and C\u00e9 cile Paris.","author":"Zhang Shiwei","year":"2020","unstructured":"Shiwei Zhang , Xiuzhen Zhang , Jey Han Lau , Jeffrey Chan, and C\u00e9 cile Paris. 2020 . Less is More : Rejecting Unreliable Reviews for Product Question Answering. CoRR , Vol. abs\/ 2007 .04526 (2020). https:\/\/arxiv.org\/abs\/2007.04526 Shiwei Zhang, Xiuzhen Zhang, Jey Han Lau, Jeffrey Chan, and C\u00e9 cile Paris. 2020. Less is More: Rejecting Unreliable Reviews for Product Question Answering. CoRR, Vol. abs\/2007.04526 (2020). https:\/\/arxiv.org\/abs\/2007.04526"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2187980.2188199"}],"event":{"name":"SIGIR '22: The 45th International ACM SIGIR Conference on Research and Development in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Madrid Spain","acronym":"SIGIR '22"},"container-title":["Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3477495.3532006","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3477495.3532006","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:10:20Z","timestamp":1750183820000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3477495.3532006"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,7,6]]},"references-count":31,"alternative-id":["10.1145\/3477495.3532006","10.1145\/3477495"],"URL":"https:\/\/doi.org\/10.1145\/3477495.3532006","relation":{},"subject":[],"published":{"date-parts":[[2022,7,6]]},"assertion":[{"value":"2022-07-07","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}