{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,29]],"date-time":"2026-07-29T14:48:15Z","timestamp":1785336495773,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":67,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T00:00:00Z","timestamp":1720569600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,10]]},"DOI":"10.1145\/3626772.3657892","type":"proceedings-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T12:40:05Z","timestamp":1720701605000},"page":"1546-1556","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":16,"title":["Unbiased Learning to Rank Meets Reality: Lessons from Baidu's Large-Scale Search Dataset"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5696-9732","authenticated-orcid":false,"given":"Philipp","family":"Hager","sequence":"first","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8265-9092","authenticated-orcid":false,"given":"Romain","family":"Deffayet","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7516-3707","authenticated-orcid":false,"given":"Jean-Michel","family":"Renders","sequence":"additional","affiliation":[{"name":"Naver Labs Europe, Meylan, France"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1704-706X","authenticated-orcid":false,"given":"Onno","family":"Zoeter","sequence":"additional","affiliation":[{"name":"Booking.com, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1086-0202","authenticated-orcid":false,"given":"Maarten","family":"de Rijke","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Addressing Trust Bias for Unbiased Learning-to-Rank. In The World Wide Web Conference (WWW).","author":"Agarwal Aman","year":"2019","unstructured":"Aman Agarwal, Xuanhui Wang, Cheng Li, Michael Bendersky, and Marc Najork. 2019a. Addressing Trust Bias for Unbiased Learning-to-Rank. In The World Wide Web Conference (WWW)."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3289600.3291017"},{"key":"e_1_3_2_1_3_1","volume-title":"The 41st International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR).","author":"Ai Qingyao","unstructured":"Qingyao Ai, Keping Bi, Cheng Luo, Jiafeng Guo, and W. Bruce Croft. 2018a. Unbiased Learning to Rank with Unbiased Propensity Estimation. In The 41st International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR)."},{"key":"e_1_3_2_1_4_1","volume-title":"The 41st International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR).","author":"Ai Qingyao","unstructured":"Qingyao Ai, Keping Bi, Cheng Luo, Jiafeng Guo, and W. Bruce Croft. 2018b. Unbiased Learning to Rank with Unbiased Propensity Estimation. In The 41st International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR)."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3439861"},{"key":"e_1_3_2_1_6_1","volume-title":"Machine Learning and Knowledge Discovery in Databases: European Conference (ECML PKDD).","author":"Bekker Jessa","year":"2019","unstructured":"Jessa Bekker, Pieter Robberechts, and Jesse Davis. 2019. Beyond the Selected Completely at Random Assumption for Learning from Positive and Unlabeled Data. In Machine Learning and Knowledge Discovery in Databases: European Conference (ECML PKDD)."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2872427.2883033"},{"key":"e_1_3_2_1_8_1","volume-title":"Chris Leary, Dougal Maclaurin, George Necula, Adam Paszke, Jake VanderPlas, Skye Wanderman-Milne, and Qiao Zhang.","author":"Bradbury James","year":"2018","unstructured":"James Bradbury, Roy Frostig, Peter Hawkins, Matthew James Johnson, Chris Leary, Dougal Maclaurin, George Necula, Adam Paszke, Jake VanderPlas, Skye Wanderman-Milne, and Qiao Zhang. 2018. JAX: composable transformations of PythonNumPy programs. http:\/\/github.com\/google\/jax"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341981.3344221"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.5555\/2976456.2976481"},{"key":"e_1_3_2_1_11_1","first-page":"1","article-title":"Yahoo! Learning to Rank Challenge Overview","volume":"14","author":"Chapelle Olivier","year":"2011","unstructured":"Olivier Chapelle and Yi Chang. 2011. Yahoo! Learning to Rank Challenge Overview. Journal of Machine Learning Research (JMLR) , Vol. 14 (2011), 1--24.","journal-title":"Journal of Machine Learning Research (JMLR)"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/1526709.1526711"},{"key":"e_1_3_2_1_13_1","volume-title":"Proceedings of The Sixteen ACM International Conference on Web Search and Data Mining (WSDM).","author":"Chen Jia","year":"2023","unstructured":"Jia Chen, Haitao Li, Weihang Su, Qingyao Ai, and Yiqun Liu. 2023 a. THUIR at WSDM Cup 2023 Task 1: Unbiased Learning to Rank. In Proceedings of The Sixteen ACM International Conference on Web Search and Data Mining (WSDM)."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371819"},{"key":"e_1_3_2_1_15_1","volume-title":"Identifiability Matters: Revealing the Hidden Recoverable Condition in Unbiased Learning to Rank. arxiv: 2309.15560 [cs.IR]","author":"Chen Mouxiang","year":"2024","unstructured":"Mouxiang Chen, Chenghao Liu, Zemin Liu, Zhuo Li, and Jianling Sun. 2024. Identifiability Matters: Revealing the Hidden Recoverable Condition in Unbiased Learning to Rank. arxiv: 2309.15560 [cs.IR]"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1006\/csla.1999.0128"},{"key":"e_1_3_2_1_17_1","volume-title":"Proceedings of The Sixteen ACM International Conference on Web Search and Data Mining (WSDM).","author":"Chen Xiaoshu","year":"2023","unstructured":"Xiaoshu Chen, Xiangsheng Li, Kunliang Wei, Bin Hu, Lei Jiang, Zeqian Huang, and Zhanhui Kang. 2023 b. Multi-Feature Integration for Perception-Dependent Examination-Bias Estimation. In Proceedings of The Sixteen ACM International Conference on Web Search and Data Mining (WSDM)."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","unstructured":"Aleksandr Chuklin Ilya Markov and Maarten de Rijke. 2015. Click Models for Web Search. Morgan & Claypool. https:\/\/doi.org\/10.2200\/S00654ED1V01Y201507ICR043","DOI":"10.2200\/S00654ED1V01Y201507ICR043"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/1341531.1341545"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2987380"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531740"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539618.3591639"},{"key":"e_1_3_2_1_23_1","article-title":"b. Evaluating the Robustness of Click Models to Policy Distributional Shift","volume":"41","author":"Deffayet Romain","year":"2023","unstructured":"Romain Deffayet, Jean-Michel Renders, and Maarten De Rijke. 2023 b. Evaluating the Robustness of Click Models to Policy Distributional Shift. ACM Transactions on Information Systems (TOIS), Vol. 41, 4, Article 84 (2023).","journal-title":"ACM Transactions on Information Systems (TOIS)"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.2517-6161.1977.tb01600.x"},{"key":"e_1_3_2_1_25_1","volume-title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. arxiv","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. arxiv: 1810.04805 [cs.CL]"},{"key":"e_1_3_2_1_26_1","volume-title":"International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR).","author":"Georges","unstructured":"Georges E. Dupret and Benjamin Piwowarski. 2008. A User Browsing Model to Predict Search Engine Click Data from Past Observations.. In International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR)."},{"key":"e_1_3_2_1_27_1","unstructured":"Varun Godbole George E. Dahl Justin Gilmer Christopher J. Shallue and Zachary Nado. 2023. Deep Learning Tuning Playbook. http:\/\/github.com\/google-research\/tuning_playbook Version 1.0."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3298689.3347033"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3616855.3636451"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539618.3591760"},{"key":"e_1_3_2_1_31_1","volume-title":"Unbiased LambdaMART: An Unbiased Pairwise Learning-to-Rank Algorithm. In The World Wide Web Conference (WWW).","author":"Hu Ziniu","year":"2019","unstructured":"Ziniu Hu, Yang Wang, Qu Peng, and Hang Li. 2019. Unbiased LambdaMART: An Unbiased Pairwise Learning-to-Rank Algorithm. In The World Wide Web Conference (WWW)."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3534678.3539065"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/1076034.1076063"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3018661.3018699"},{"key":"e_1_3_2_1_35_1","volume-title":"Proceedings of The Sixteen ACM International Conference on Web Search and Data Mining (WSDM).","author":"Li Haitao","year":"2023","unstructured":"Haitao Li, Jia Chen, Weihang Su, Qingyao Ai, and Yiqun Liu. 2023 a. Towards Better Web Search Performance: Pre-training, Fine-tuning and Learning to Rank. In Proceedings of The Sixteen ACM International Conference on Web Search and Data Mining (WSDM)."},{"key":"e_1_3_2_1_36_1","volume-title":"Proceedings of The Sixteen ACM International Conference on Web Search and Data Mining (WSDM).","author":"Li Xiangsheng","year":"2023","unstructured":"Xiangsheng Li, Xiaoshu Chen, Kunliang Wei, Bin Hu, Lei Jiang, Zeqian Huang, and Zhanhui Kang. 2023 b. Pretraining De-Biased Language Model with Large-scale Click Logs for Document Ranking. In Proceedings of The Sixteen ACM International Conference on Web Search and Data Mining (WSDM)."},{"key":"e_1_3_2_1_37_1","unstructured":"Ilya Loshchilov and Frank Hutter. 2019. Decoupled Weight Decay Regularization. (2019). arxiv: 1711.05101 [cs.LG]"},{"key":"e_1_3_2_1_38_1","volume-title":"The 17th Round of NII Testbeds and Community for Information Access Research (NTCIR).","author":"Niu Zechun","unstructured":"Zechun Niu, Jiaxin Mao, Qingyao Ai, Lixin Zou, Shuaiqiang Wang, and Dawei Yin. 2023. Overview of the NTCIR-17 Unbiased Learning to Rank Evaluation 2 (ULTRE-2) Task. In The 17th Round of NII Testbeds and Community for Information Access Research (NTCIR)."},{"key":"e_1_3_2_1_39_1","volume-title":"Multi-Stage Document Ranking with BERT. arxiv","author":"Nogueira Rodrigo","year":"1910","unstructured":"Rodrigo Nogueira, Wei Yang, Kyunghyun Cho, and Jimmy Lin. 2019. Multi-Stage Document Ranking with BERT. arxiv: 1910.14424 [cs.IR]"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539813.3545137"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3437963.3441794"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330677"},{"key":"e_1_3_2_1_43_1","unstructured":"Ronak Pradeep Rodrigo Nogueira and Jimmy Lin. 2021. The Expando-Mono-Duo Design Pattern for Text Ranking with Pretrained Sequence-to-Sequence Models. arxiv: 2101.05667 [cs.IR]"},{"key":"e_1_3_2_1_44_1","unstructured":"Tao Qin and Tie-Yan Liu. 2013. Introducing LETOR 4.0 Datasets. arxiv: 1306.2597 [cs.IR]"},{"key":"e_1_3_2_1_45_1","volume-title":"International Conference on Learning Representations (ICLR).","author":"Qin Zhen","year":"2021","unstructured":"Zhen Qin, Le Yan, Honglei Zhuang, Yi Tay, Rama Kumar Pasumarthi, Xuanhui Wang, Mike Bendersky, and Marc Najork. 2021. Are Neural Rankers still Outperformed by Gradient Boosted Decision Trees?. In International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/1242572.1242643"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1108\/00220410410560582"},{"key":"e_1_3_2_1_48_1","volume-title":"Proceedings of The Third Text REtrieval Conference, TREC (NIST Special Publication","volume":"126","author":"Robertson Stephen E.","year":"1994","unstructured":"Stephen E. Robertson, Steve Walker, Susan Jones, Micheline Hancock-Beaulieu, and Mike Gatford. 1994. Okapi at TREC-3. In Proceedings of The Third Text REtrieval Conference, TREC (NIST Special Publication, Vol. 500--225). 109--126."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371783"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000009"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539618.3591745"},{"key":"e_1_3_2_1_52_1","volume-title":"Proceedings of The Sixteen ACM International Conference on Web Search and Data Mining (WSDM).","author":"Sun Xiaojie","year":"2023","unstructured":"Xiaojie Sun, Lulu Yu, Yiting Wang, Keping Bi, and Jiafeng Guo. 2023. Ensemble Ranking Model with Multiple Pretraining Strategies for Web Search. In Proceedings of The Sixteen ACM International Conference on Web Search and Data Mining (WSDM)."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3459637.3482020"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401299"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3412031"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911537"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159652.3159732"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531837"},{"key":"e_1_3_2_1_59_1","volume-title":"The 17th Round of NII Testbeds and Community for Information Access Research (NTCIR).","author":"Yu Lulu","unstructured":"Lulu Yu, Keping Bi, Jiafeng Guo, and Xueqi Cheng. 2023 a. CIR at the NTCIR-17 ULTRE-2 Task. In The 17th Round of NII Testbeds and Community for Information Access Research (NTCIR)."},{"key":"e_1_3_2_1_60_1","volume-title":"Proceedings of The Sixteen ACM International Conference on Web Search and Data Mining (WSDM).","author":"Yu Lulu","year":"2023","unstructured":"Lulu Yu, Yiting Wang, Xiaojie Sun, Keping Bi, and Jiafeng Guo. 2023 b. Feature-Enhanced Network with Hybrid Debiasing Strategies for Unbiased Learning to Rank. In Proceedings of The Sixteen ACM International Conference on Web Search and Data Mining (WSDM)."},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3580305.3599914"},{"key":"e_1_3_2_1_62_1","volume-title":"The 16th Round of NII Testbeds and Community for Information Access Research (NTCIR).","author":"Zhao Yurou","unstructured":"Yurou Zhao, Zechun Niu, Feng Wang, Jiaxin Mao, Qingyao Ai, Yang Tao, Junqi Zhang, and Yiqun Liu. 2022. Overview of the NTCIR-16 Unbiased Learning to Rank Evaluation (ULTRE) Task. In The 16th Round of NII Testbeds and Community for Information Access Research (NTCIR)."},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/3298689.3346997"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3450098"},{"key":"e_1_3_2_1_65_1","unstructured":"Lixin Zou Haitao Mao Xiaokai Chu Jiliang Tang Wenwen Ye Shuaiqiang Wang and Dawei Yin. 2022. A Large Scale Search Dataset for Unbiased Learning to Rank. In Advances in Neural Information Processing Systems (NeurIPS)."},{"key":"e_1_3_2_1_66_1","unstructured":"Lixin Zou Haitao Mao Xiaokai Chu Wenwen Ye Changying Hao Shuaiqiang Wang Dawei Yin Jiliang Tang Aixin Sun and Ce Zhang. 2023 a. Pre-training for Web Search. https:\/\/aistudio.baidu.com\/competition\/detail\/536\/0\/introduction"},{"key":"e_1_3_2_1_67_1","unstructured":"Lixin Zou Haitao Mao Xiaokai Chu Wenwen Ye Changying Hao Shuaiqiang Wang Dawei Yin Jiliang Tang Aixin Sun and Ce Zhang. 2023 b. Unbiased Learning for Web Search. https:\/\/aistudio.baidu.com\/competition\/detail\/534\/0\/introduction"}],"event":{"name":"SIGIR 2024: The 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Washington DC USA","acronym":"SIGIR 2024","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657892","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3626772.3657892","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:39:24Z","timestamp":1755841164000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657892"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,10]]},"references-count":67,"alternative-id":["10.1145\/3626772.3657892","10.1145\/3626772"],"URL":"https:\/\/doi.org\/10.1145\/3626772.3657892","relation":{},"subject":[],"published":{"date-parts":[[2024,7,10]]},"assertion":[{"value":"2024-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}