{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,9]],"date-time":"2026-05-09T03:33:35Z","timestamp":1778297615129,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":77,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,7,18]],"date-time":"2023-07-18T00:00:00Z","timestamp":1689638400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"European Union's Horizon Europe","award":["101070212"],"award-info":[{"award-number":["101070212"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,7,19]]},"DOI":"10.1145\/3539618.3591933","type":"proceedings-article","created":{"date-parts":[[2023,7,19]],"date-time":"2023-07-19T00:22:59Z","timestamp":1689726179000},"page":"2650-2660","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":9,"title":["The Role of Relevance in Fair Ranking"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1621-9536","authenticated-orcid":false,"given":"Aparna","family":"Balagopalan","sequence":"first","affiliation":[{"name":"Massachusetts Institute of Technology, Cambridge, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6452-4386","authenticated-orcid":false,"given":"Abigail Z.","family":"Jacobs","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, MI, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8083-0976","authenticated-orcid":false,"given":"Asia J.","family":"Biega","sequence":"additional","affiliation":[{"name":"Max Planck Institute for Security and Privacy, Bochum, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,7,18]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"crossref","unstructured":"Aman Agarwal Xuanhui Wang Cheng Li Michael Bendersky and Marc Najork. 2019. Addressing trust bias for unbiased learning-to-rank. In WWW. 4--14.","DOI":"10.1145\/3308558.3313697"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"crossref","unstructured":"Qingyao Ai Keping Bi Jiafeng Guo and W Bruce Croft. 2018a. Learning a deep listwise context model for ranking refinement. In ACM SIGIR. 135--144.","DOI":"10.1145\/3209978.3209985"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"crossref","unstructured":"Qingyao Ai Keping Bi Cheng Luo Jiafeng Guo and W Bruce Croft. 2018b. Unbiased learning to rank with unbiased propensity estimation. In SIGIR. 385--394.","DOI":"10.1145\/3209978.3209986"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"crossref","unstructured":"Abolfazl Asudeh HV Jagadish Julia Stoyanovich and Gautam Das. 2019. Designing fair ranking schemes. In SIGMOD. 1259--1276.","DOI":"10.1145\/3299869.3300079"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"crossref","unstructured":"Peter Bailey Nick Craswell Ian Soboroff Paul Thomas Arjen P de Vries and Emine Yilmaz. 2008. Relevance assessment: are judges exchangeable and does it matter. In SIGIR. 667--674.","DOI":"10.1145\/1390334.1390447"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-45442-5_26"},{"key":"e_1_3_2_2_7_1","unstructured":"Solon Barocas Moritz Hardt and Arvind Narayanan. 2019. Fairness and Machine Learning. fairmlbook. org."},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330745"},{"key":"e_1_3_2_2_9_1","volume-title":"Overview of the TREC 2020 Fair Ranking Track. arXiv preprint arXiv:2108","author":"Biega Asia J","year":"2021","unstructured":"Asia J Biega, Fernando Diaz, Michael D Ekstrand, Sergey Feldman, and Sebastian Kohlmeier. 2021. Overview of the TREC 2020 Fair Ranking Track. arXiv preprint arXiv:2108.05135 (2021)."},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"crossref","unstructured":"Asia J Biega Krishna P Gummadi and Gerhard Weikum. 2018. Equity of attention: Amortizing individual fairness in rankings. In SIGIR. 405--414.","DOI":"10.1145\/3209978.3210063"},{"key":"e_1_3_2_2_11_1","volume-title":"Understanding inequalities in ride-hailing services through simulations. Scientific reports","author":"Bok\u00e1nyi Eszter","year":"2020","unstructured":"Eszter Bok\u00e1nyi and Anik\u00f3 Hann\u00e1k. 2020. Understanding inequalities in ride-hailing services through simulations. Scientific reports, Vol. 10, 1 (2020), 1--11."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.10286"},{"key":"e_1_3_2_2_13_1","unstructured":"Amanda Bower Hamid Eftekhari Mikhail Yurochkin and Yuekai Sun. 2020. Individually Fair Rankings. In ICLR."},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308774.3308783"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijhm.2017.11.002"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-02294-4"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.cor.2004.09.027"},{"key":"e_1_3_2_2_18_1","first-page":"870","article-title":"Tackling unconscious bias in hiring practices: The plight of the Rooney rule","volume":"82","author":"Collins Brian W","year":"2007","unstructured":"Brian W Collins. 2007. Tackling unconscious bias in hiring practices: The plight of the Rooney rule. NYUL Rev., Vol. 82 (2007), 870.","journal-title":"NYUL Rev."},{"key":"e_1_3_2_2_19_1","volume-title":"Christopher De Sa, and Siddhartha Sen","author":"Cooper A Feder","year":"2023","unstructured":"A Feder Cooper, Solon Barocas, Christopher De Sa, and Siddhartha Sen. 2023. Variance, Self-Consistency, and Arbitrariness in Fair Classification. arXiv:2301.11562 (2023)."},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1016\/0020-0271(73)90004-1"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462804"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"crossref","unstructured":"Nick Craswell Onno Zoeter Michael Taylor and Bill Ramsey. 2008. An experimental comparison of click position-bias models. In WSDM. 87--94.","DOI":"10.1145\/1341531.1341545"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1108\/eb026436"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"crossref","unstructured":"Fernando Diaz Bhaskar Mitra Michael D Ekstrand Asia J Biega and Ben Carterette. 2020. Evaluating stochastic rankings with expected exposure. In CIKM. 275--284.","DOI":"10.1145\/3340531.3411962"},{"key":"e_1_3_2_2_25_1","volume-title":"SimuRec: Workshop on Synthetic Data and Simulation Methods for Recommender Systems Research. In RecSys. 803--805","author":"Ekstrand Michael D","year":"2021","unstructured":"Michael D Ekstrand, Allison Chaney, Pablo Castells, Robin Burke, David Rohde, and Manel Slokom. 2021. SimuRec: Workshop on Synthetic Data and Simulation Methods for Recommender Systems Research. In RecSys. 803--805."},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000079"},{"key":"e_1_3_2_2_27_1","volume-title":"Overview of the TREC 2021 Fair Ranking Track. In The Thirtieth Text REtrieval Conference (TREC 2021) Proceedings.","author":"Ekstrand Michael D.","year":"2022","unstructured":"Michael D. Ekstrand, Graham McDonald, Amifa Raj, and Isaac Johnson. 2022b. Overview of the TREC 2021 Fair Ranking Track. In The Thirtieth Text REtrieval Conference (TREC 2021) Proceedings."},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1609\/icwsm.v14i1.7288"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jhtm.2020.01.005"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-85616-8_33"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330691"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"crossref","unstructured":"Avijit Ghosh Ritam Dutt and Christo Wilson. 2021. When fair ranking meets uncertain inference. In SIGIR. 1033--1043.","DOI":"10.1145\/3404835.3462850"},{"key":"e_1_3_2_2_33_1","unstructured":"Ziniu Hu Yang Wang Qu Peng and Hang Li. 2019. Unbiased LambdaMART: An unbiased pairwise learning-to-rank algorithm. In WWW. 2830--2836."},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445901"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3130348.3130374"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3130332.3130334"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/1229179.1229181"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v42i3.18141"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"crossref","unstructured":"Thorsten Joachims Adith Swaminathan and Tobias Schnabel. 2017b. Unbiased learning-to-rank with biased feedback. In WSDM. 781--789.","DOI":"10.1145\/3018661.3018699"},{"key":"e_1_3_2_2_40_1","volume-title":"MEDINFO","author":"Kagolovsky Yuri","year":"2001","unstructured":"Yuri Kagolovsky and Jochen R M\u00f6hr. 2001. A new approach to the concept of ?relevance\" in information retrieval (IR). In MEDINFO 2001. IOS Press, 348--352."},{"key":"e_1_3_2_2_41_1","volume-title":"Binary and graded relevance in IR evaluations-comparison of the effects on ranking of IR systems. Information processing & management","author":"Kek\u00e4l\u00e4inen Jaana","year":"2005","unstructured":"Jaana Kek\u00e4l\u00e4inen. 2005. Binary and graded relevance in IR evaluations-comparison of the effects on ranking of IR systems. Information processing & management, Vol. 41, 5 (2005), 1019--1033."},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3450080"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3532036"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1951.10500769"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3272027"},{"key":"e_1_3_2_2_46_1","volume-title":"Impact of query sample selection bias on information retrieval system ranking","author":"Melucci Massimo","unstructured":"Massimo Melucci. 2016. Impact of query sample selection bias on information retrieval system ranking. In DSAA. IEEE, 341--350."},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"crossref","unstructured":"Marco Morik Ashudeep Singh Jessica Hong and Thorsten Joachims. 2020. Controlling fairness and bias in dynamic learning-to-rank. In SIGIR. 429--438.","DOI":"10.1145\/3397271.3401100"},{"key":"e_1_3_2_2_48_1","volume-title":"CPFair: Personalized Consumer and Producer Fairness Re-Ranking for Recommender Systems (SIGIR '22)","author":"Naghiaei Mohammadmehdi","year":"2022","unstructured":"Mohammadmehdi Naghiaei, Hossein A. Rahmani, and Yashar Deldjoo. 2022. CPFair: Personalized Consumer and Producer Fairness Re-Ranking for Recommender Systems (SIGIR '22). 770--779."},{"key":"e_1_3_2_2_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539813.3545137"},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"crossref","unstructured":"Harrie Oosterhuis and Maarten de Rijke. 2018. Differentiable unbiased online learning to rank. In CIKM. 1293--1302.","DOI":"10.1145\/3269206.3271686"},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533238"},{"key":"e_1_3_2_2_52_1","volume-title":"Algorithmic neutrality. arXiv preprint arXiv:2303.05103","author":"Phillips-Brown Milo","year":"2023","unstructured":"Milo Phillips-Brown. 2023. Algorithmic neutrality. arXiv preprint arXiv:2303.05103 (2023)."},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3532018"},{"key":"e_1_3_2_2_54_1","volume-title":"The probability ranking principle in IR. J Doc","author":"Robertson Stephen E","year":"1977","unstructured":"Stephen E Robertson. 1977. The probability ranking principle in IR. J Doc (1977)."},{"key":"e_1_3_2_2_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308560.3317595"},{"key":"e_1_3_2_2_56_1","volume-title":"The Notion of Relevance in Information Science: Everybody knows what relevance is. But, what is it really? Synthesis lectures on information concepts, retrieval, and services","author":"Saracevic Tefko","year":"2016","unstructured":"Tefko Saracevic. 2016. The Notion of Relevance in Information Science: Everybody knows what relevance is. But, what is it really? Synthesis lectures on information concepts, retrieval, and services, Vol. 8, 3 (2016), i--109."},{"key":"e_1_3_2_2_57_1","unstructured":"Tobias Schnabel Adith Swaminathan Ashudeep Singh Navin Chandak and Thorsten Joachims. 2016. Recommendations as treatments: Debiasing learning and evaluation. In ICML. 1670--1679."},{"key":"e_1_3_2_2_58_1","volume-title":"Properties of Group Fairness Metrics for Rankings. arXiv preprint arXiv:2212.14351","author":"Schumacher Tobias","year":"2022","unstructured":"Tobias Schumacher, Marlene Lutz, Sandipan Sikdar, and Markus Strohmaier. 2022. Properties of Group Fairness Metrics for Rankings. arXiv preprint arXiv:2212.14351 (2022)."},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"crossref","unstructured":"Anne Schuth Harrie Oosterhuis Shimon Whiteson and Maarten de Rijke. 2016. Multileave gradient descent for fast online learning to rank. In WSDM. 457--466.","DOI":"10.1145\/2835776.2835804"},{"key":"e_1_3_2_2_60_1","volume-title":"Exploring artist gender bias in music recommendation. arXiv preprint arXiv:2009.01715","author":"Shakespeare Dougal","year":"2020","unstructured":"Dougal Shakespeare, Lorenzo Porcaro, Emilia G\u00f3mez, and Carlos Castillo. 2020. Exploring artist gender bias in music recommendation. arXiv preprint arXiv:2009.01715 (2020)."},{"key":"e_1_3_2_2_61_1","doi-asserted-by":"crossref","unstructured":"Ashudeep Singh and Thorsten Joachims. 2018. Fairness of exposure in rankings. In KDD. 2219--2228.","DOI":"10.1145\/3219819.3220088"},{"key":"e_1_3_2_2_62_1","volume-title":"Policy learning for fairness in ranking. Advances in neural information processing systems","author":"Singh Ashudeep","year":"2019","unstructured":"Ashudeep Singh and Thorsten Joachims. 2019. Policy learning for fairness in ranking. Advances in neural information processing systems, Vol. 32 (2019)."},{"key":"e_1_3_2_2_63_1","volume-title":"A Pre-processing Method for Fairness in Ranking. arXiv preprint arXiv:2110.15503","author":"Sonoda Ryosuke","year":"2021","unstructured":"Ryosuke Sonoda. 2021. A Pre-processing Method for Fairness in Ranking. arXiv preprint arXiv:2110.15503 (2021)."},{"key":"e_1_3_2_2_64_1","doi-asserted-by":"crossref","unstructured":"Ali Vardasbi Maarten de Rijke and Ilya Markov. 2020. Cascade model-based propensity estimation for counterfactual learning to rank. In SIGIR. 2089--2092.","DOI":"10.1145\/3397271.3401299"},{"key":"e_1_3_2_2_65_1","volume-title":"TREC: Experiment and evaluation in information retrieval","author":"Voorhees Ellen M","year":"2005","unstructured":"Ellen M Voorhees, Donna K Harman, et al. 2005. TREC: Experiment and evaluation in information retrieval, vol. 1."},{"key":"e_1_3_2_2_66_1","doi-asserted-by":"publisher","DOI":"10.1609\/icwsm.v11i1.14888"},{"key":"e_1_3_2_2_67_1","doi-asserted-by":"crossref","unstructured":"Xuanhui Wang Michael Bendersky Donald Metzler and Marc Najork. 2016. Learning to rank with selection bias in personal search. In SIGIR. 115--124.","DOI":"10.1145\/2911451.2911537"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"crossref","unstructured":"X Wang N Golbandi M Bendersky D Metzler and M Najork. 2018. Position bias estimation for unbiased learning to rank in personal search. In WSDM.","DOI":"10.1145\/3159652.3159732"},{"key":"e_1_3_2_2_69_1","volume-title":"Harms as process: opportunity harms in the US mortgage market. preprint","author":"Xian Lu","year":"2022","unstructured":"Lu Xian and Abigail Z Jacobs. 2022. Harms as process: opportunity harms in the US mortgage market. preprint (2022)."},{"key":"e_1_3_2_2_70_1","unstructured":"Himank Yadav Zhengxiao Du and Thorsten Joachims. 2020. Fair learning-to-rank from implicit feedback. In SIGIR."},{"key":"e_1_3_2_2_71_1","volume-title":"Causal intersectionality for fair ranking. arXiv preprint arXiv:2006.08688","author":"Yang Ke","year":"2020","unstructured":"Ke Yang, Joshua R Loftus, and Julia Stoyanovich. 2020. Causal intersectionality for fair ranking. arXiv preprint arXiv:2006.08688 (2020)."},{"key":"e_1_3_2_2_72_1","doi-asserted-by":"crossref","unstructured":"Tao Yang and Qingyao Ai. 2021. Maximizing Marginal Fairness for Dynamic Learning to Rank. In WebConf. 137--145.","DOI":"10.1145\/3442381.3449901"},{"key":"e_1_3_2_2_73_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939677"},{"key":"e_1_3_2_2_74_1","doi-asserted-by":"crossref","unstructured":"Yisong Yue and Thorsten Joachims. 2009. Interactively optimizing information retrieval systems as a dueling bandits problem. In ICML. 1201--1208.","DOI":"10.1145\/1553374.1553527"},{"key":"e_1_3_2_2_75_1","volume-title":"Fairness in Ranking: A Survey. arXiv preprint arXiv:2103.14000","author":"Zehlike Meike","year":"2021","unstructured":"Meike Zehlike, Ke Yang, and Julia Stoyanovich. 2021. Fairness in Ranking: A Survey. arXiv preprint arXiv:2103.14000 (2021)."},{"key":"e_1_3_2_2_76_1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3533380","article-title":"Fairness in ranking, part ii: Learning-to-rank and recommender systems","volume":"55","author":"Zehlike Meike","year":"2022","unstructured":"Meike Zehlike, Ke Yang, and Julia Stoyanovich. 2022. Fairness in ranking, part ii: Learning-to-rank and recommender systems. Comput. Surveys, Vol. 55, 6 (2022), 1--41.","journal-title":"Comput. Surveys"},{"key":"e_1_3_2_2_77_1","unstructured":"Elana Zeide. 2022. The Silicon Ceiling: How Algorithmic Assessments Construct an Invisible Barrier to Opportunity. UMKC Law Rev. (2022)."}],"event":{"name":"SIGIR '23: The 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Taipei Taiwan","acronym":"SIGIR '23","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591933","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3539618.3591933","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:38:07Z","timestamp":1750178287000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591933"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7,18]]},"references-count":77,"alternative-id":["10.1145\/3539618.3591933","10.1145\/3539618"],"URL":"https:\/\/doi.org\/10.1145\/3539618.3591933","relation":{},"subject":[],"published":{"date-parts":[[2023,7,18]]},"assertion":[{"value":"2023-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}