{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T01:00:06Z","timestamp":1774400406508,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":80,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,4,30]],"date-time":"2023-04-30T00:00:00Z","timestamp":1682812800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,4,30]]},"DOI":"10.1145\/3543507.3583204","type":"proceedings-article","created":{"date-parts":[[2023,4,26]],"date-time":"2023-04-26T23:30:25Z","timestamp":1682551825000},"page":"3648-3659","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":23,"title":["Scoping Fairness Objectives and Identifying Fairness Metrics for Recommender Systems: The Practitioners\u2019 Perspective"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4990-912X","authenticated-orcid":false,"given":"Jessie J.","family":"Smith","sequence":"first","affiliation":[{"name":"University of Colorado, Boulder, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3351-5951","authenticated-orcid":false,"given":"Lex","family":"Beattie","sequence":"additional","affiliation":[{"name":"Spotify, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0786-0324","authenticated-orcid":false,"given":"Henriette","family":"Cramer","sequence":"additional","affiliation":[{"name":"Spotify, USA"}]}],"member":"320","published-online":{"date-parts":[[2023,4,30]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"Julia Angwin and Terry\u00a0Jr. Parris. 2016. Facebook lets advertisers exclude users by race. https:\/\/www.propublica.org\/article\/facebook-lets-advertisers-exclude-users-by-race"},{"key":"e_1_3_2_2_2_1","unstructured":"Solon Barocas Moritz Hardt and Arvind Narayanan. 2019. Fairness and Machine Learning. fairmlbook.org. http:\/\/www.fairmlbook.org."},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3523227.3547403"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1147\/JRD.2019.2942287"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330745"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372864"},{"key":"e_1_3_2_2_7_1","volume-title":"Fairlearn: A toolkit for assessing and improving fairness in AI. Microsoft, Tech. Rep. MSR-TR-2020-32","author":"Bird Sarah","year":"2020","unstructured":"Sarah Bird, Miro Dud\u00edk, Richard Edgar, Brandon Horn, Roman Lutz, Vanessa Milan, Mehrnoosh Sameki, Hanna Wallach, and Kathleen Walker. 2020. Fairlearn: A toolkit for assessing and improving fairness in AI. Microsoft, Tech. Rep. MSR-TR-2020-32 (2020)."},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","unstructured":"Amanda Bower Kristian Lum Tomo Lazovich Kyra Yee and Luca Belli. 2022. Random Isn\u2019t Always Fair: Candidate Set Imbalance and Exposure Inequality in Recommender Systems. (2022). https:\/\/doi.org\/10.48550\/ARXIV.2209.05000","DOI":"10.48550\/ARXIV.2209.05000"},{"key":"e_1_3_2_2_9_1","volume-title":"Multisided fairness for recommendation. arXiv preprint arXiv:1707.00093","author":"Burke Robin","year":"2017","unstructured":"Robin Burke. 2017. Multisided fairness for recommendation. arXiv preprint arXiv:1707.00093 (2017)."},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858498"},{"key":"e_1_3_2_2_11_1","volume-title":"Thematic analysis. Qualitative psychology: A practical guide to research methods 3","author":"Clarke Victoria","year":"2015","unstructured":"Victoria Clarke, Virginia Braun, and Nikki Hayfield. 2015. Thematic analysis. Qualitative psychology: A practical guide to research methods 3 (2015), 222\u2013248."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3278156"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3278156"},{"key":"e_1_3_2_2_14_1","volume-title":"Conference on Neural Information Processing Systems, invited speaker.","author":"Crawford Kate","year":"2017","unstructured":"Kate Crawford. 2017. The trouble with bias. In Conference on Neural Information Processing Systems, invited speaker."},{"key":"e_1_3_2_2_15_1","volume-title":"A Survey of Research on Fair Recommender Systems. arXiv preprint arXiv:2205.11127","author":"Deldjoo Yashar","year":"2022","unstructured":"Yashar Deldjoo, Dietmar Jannach, Alejandro Bellogin, Alessandro Difonzo, and Dario Zanzonelli. 2022. A Survey of Research on Fair Recommender Systems. arXiv preprint arXiv:2205.11127 (2022)."},{"key":"e_1_3_2_2_16_1","volume-title":"Jatinder Singh, Zhiwei\u00a0Steven Wu, Kenneth Holstein, and Haiyi Zhu.","author":"Deng Wesley\u00a0Hanwen","year":"2022","unstructured":"Wesley\u00a0Hanwen Deng, Manish Nagireddy, Michelle Seng\u00a0Ah Lee, Jatinder Singh, Zhiwei\u00a0Steven Wu, Kenneth Holstein, and Haiyi Zhu. 2022. Exploring How Machine Learning Practitioners (Try To) Use Fairness Toolkits. arXiv preprint arXiv:2205.06922 (2022)."},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2090236.2090255"},{"key":"e_1_3_2_2_18_1","volume-title":"Fairness in Information Access Systems. Foundations and Trends\u00ae in Information Retrieval 16, 1-2","author":"Ekstrand D","year":"2022","unstructured":"Michael\u00a0D Ekstrand, Anubrata Das, Robin Burke, Fernando Diaz, 2022. Fairness in Information Access Systems. Foundations and Trends\u00ae in Information Retrieval 16, 1-2 (2022), 1\u2013177."},{"key":"e_1_3_2_2_19_1","volume-title":"Proceedings of the machine learning: the debates workshop.","author":"Green Ben","year":"2018","unstructured":"Ben Green and Lily Hu. 2018. The myth in the methodology: Towards a recontextualization of fairness in machine learning. In Proceedings of the machine learning: the debates workshop."},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300830"},{"key":"e_1_3_2_2_21_1","unstructured":"HRW. 2018. \" Only Men Need Apply\": Gender Discrimination in Job Advertisements in China. Human Rights Watch."},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445901"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3306618.3314288"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-019-0088-2"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3461702.3462588"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313443"},{"key":"e_1_3_2_2_27_1","volume-title":"Designing Tools for Semi-Automated Detection of Machine Learning Biases: An Interview Study. arXiv preprint arXiv:2003.07680","author":"Law Po-Ming","year":"2020","unstructured":"Po-Ming Law, Sana Malik, Fan Du, and Moumita Sinha. 2020. Designing Tools for Semi-Automated Detection of Machine Learning Biases: An Interview Study. arXiv preprint arXiv:2003.07680 (2020)."},{"key":"e_1_3_2_2_28_1","volume-title":"Measuring disparate outcomes of content recommendation algorithms with distributional inequality metrics. arXiv preprint arXiv:2202.01615","author":"Lazovich Tomo","year":"2022","unstructured":"Tomo Lazovich, Luca Belli, Aaron Gonzales, Amanda Bower, Uthaipon Tantipongpipat, Kristian Lum, Ferenc Huszar, and Rumman Chowdhury. 2022. Measuring disparate outcomes of content recommendation algorithms with distributional inequality metrics. arXiv preprint arXiv:2202.01615 (2022)."},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1177\/2053951718756684"},{"key":"e_1_3_2_2_30_1","volume-title":"From fairness metrics to key ethics indicators (keis): a context-aware approach to algorithmic ethics in an unequal society. Available at SSRN","author":"Seng\u00a0Ah Lee Michelle","year":"2020","unstructured":"Michelle Seng\u00a0Ah Lee, Luciano Floridi, and Jatinder Singh. 2020. From fairness metrics to key ethics indicators (keis): a context-aware approach to algorithmic ethics in an unequal society. Available at SSRN (2020)."},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445261"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3298689.3347016"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3512899"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376445"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1353\/sor.2019.0022"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445933"},{"key":"e_1_3_2_2_37_1","volume-title":"Assembling accountability: algorithmic impact assessment for the public interest. Available at SSRN 3877437","author":"Moss Emanuel","year":"2021","unstructured":"Emanuel Moss, Elizabeth\u00a0Anne Watkins, Ranjit Singh, Madeleine\u00a0Clare Elish, and Jacob Metcalf. 2021. Assembling accountability: algorithmic impact assessment for the public interest. Available at SSRN 3877437 (2021)."},{"key":"e_1_3_2_2_38_1","volume-title":"Proc. Conf. Fairness Accountability Transp.","author":"Narayanan Arvind","year":"2018","unstructured":"Arvind Narayanan. 2018. Translation tutorial: 21 fairness definitions and their politics. In Proc. Conf. Fairness Accountability Transp., New York, USA, Vol.\u00a01170. 3."},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287567"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274405"},{"key":"e_1_3_2_2_41_1","volume-title":"Fair ranking: a critical review, challenges, and future directions. arXiv preprint arXiv:2201.12662","author":"Patro K","year":"2022","unstructured":"Gourab\u00a0K Patro, Lorenzo Porcaro, Laura Mitchell, Qiuyue Zhang, Meike Zehlike, and Nikhil Garg. 2022. Fair ranking: a critical review, challenges, and future directions. arXiv preprint arXiv:2201.12662 (2022)."},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3449081"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372879"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445604"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1177\/1468794119884806"},{"key":"e_1_3_2_2_46_1","volume-title":"Aequitas: A bias and fairness audit toolkit. arXiv preprint arXiv:1811.05577","author":"Saleiro Pedro","year":"2018","unstructured":"Pedro Saleiro, Benedict Kuester, Loren Hinkson, Jesse London, Abby Stevens, Ari Anisfeld, Kit\u00a0T Rodolfa, and Rayid Ghani. 2018. Aequitas: A bias and fairness audit toolkit. arXiv preprint arXiv:1811.05577 (2018)."},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341164"},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287598"},{"key":"e_1_3_2_2_49_1","volume-title":"Exploring user opinions of fairness in recommender systems. arXiv preprint arXiv:2003.06461","author":"Smith Jessie","year":"2020","unstructured":"Jessie Smith, Nasim Sonboli, Casey Fiesler, and Robin Burke. 2020. Exploring user opinions of fairness in recommender systems. arXiv preprint arXiv:2003.06461 (2020)."},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3383313.3411525"},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340631.3394846"},{"key":"e_1_3_2_2_52_1","doi-asserted-by":"publisher","DOI":"10.1177\/2053951717743530"},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-52485-2_1"},{"key":"e_1_3_2_2_54_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.clsr.2021.105567"},{"key":"e_1_3_2_2_55_1","volume-title":"Fairness indicators: scalable infrastructure for fair ML system. Mountain View (CA): Google (accessed 2020-01-27). https:\/\/ai. googleblog. com\/2019\/12\/fairness-indicators-scalable. html","author":"Xu C","year":"2019","unstructured":"C Xu and T Doshi. 2019. Fairness indicators: scalable infrastructure for fair ML system. Mountain View (CA): Google (accessed 2020-01-27). https:\/\/ai. googleblog. com\/2019\/12\/fairness-indicators-scalable. html (2019)."},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132847.3132938"},{"key":"e_1_3_2_2_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359301"},{"key":"e_1_3_2_2_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3299869.3300079"},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330745"},{"key":"e_1_3_2_2_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210063"},{"key":"e_1_3_2_2_61_1","volume-title":"A conceptual framework for evaluating fairness in search. arXiv preprint arXiv:1907.09328","author":"Das Anubrata","year":"2019","unstructured":"Anubrata Das and Matthew Lease. 2019. A conceptual framework for evaluating fairness in search. arXiv preprint arXiv:1907.09328 (2019)."},{"key":"e_1_3_2_2_62_1","volume-title":"A Survey of Research on Fair Recommender Systems. arXiv preprint arXiv:2205.11127","author":"Deldjoo Yashar","year":"2022","unstructured":"Yashar Deldjoo, Dietmar Jannach, Alejandro Bellogin, Alessandro Difonzo, and Dario Zanzonelli. 2022. A Survey of Research on Fair Recommender Systems. arXiv preprint arXiv:2205.11127 (2022)."},{"key":"e_1_3_2_2_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3411962"},{"key":"e_1_3_2_2_64_1","volume-title":"Conference on fairness, accountability and transparency. PMLR, 172\u2013186","author":"Ekstrand D","year":"2018","unstructured":"Michael\u00a0D Ekstrand, Mucun Tian, Ion\u00a0Madrazo Azpiazu, Jennifer\u00a0D Ekstrand, Oghenemaro Anuyah, David McNeill, and Maria\u00a0Soledad Pera. 2018. All the cool kids, how do they fit in?: Popularity and demographic biases in recommender evaluation and effectiveness. In Conference on fairness, accountability and transparency. PMLR, 172\u2013186."},{"key":"e_1_3_2_2_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330691"},{"key":"e_1_3_2_2_66_1","volume-title":"International Conference on Machine Learning. PMLR","author":"Hashimoto Tatsunori","year":"2018","unstructured":"Tatsunori Hashimoto, Megha Srivastava, Hongseok Namkoong, and Percy Liang. 2018. Fairness without demographics in repeated loss minimization. In International Conference on Machine Learning. PMLR, 1929\u20131938."},{"key":"e_1_3_2_2_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313443"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3449866"},{"key":"e_1_3_2_2_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462966"},{"key":"e_1_3_2_2_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/3298689.3347016"},{"key":"e_1_3_2_2_71_1","doi-asserted-by":"publisher","unstructured":"Rishabh Mehrotra Ashton Anderson Fernando Diaz Amit Sharma Hanna Wallach and Emine Yilmaz. 2017. Auditing Search Engines for Differential Satisfaction Across Demographics. In Proceedings of the 26th International Conference on World Wide Web Companion (Perth Australia) (WWW \u201917 Companion). International World Wide Web Conferences Steering Committee Republic and Canton of Geneva CHE 626\u2013633. https:\/\/doi.org\/10.1145\/3041021.3054197","DOI":"10.1145\/3041021.3054197"},{"key":"e_1_3_2_2_72_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308560.3317595"},{"key":"e_1_3_2_2_73_1","doi-asserted-by":"publisher","DOI":"10.1145\/3219819.3220088"},{"key":"e_1_3_2_2_74_1","doi-asserted-by":"publisher","DOI":"10.1145\/3240323.3240372"},{"key":"e_1_3_2_2_75_1","doi-asserted-by":"publisher","DOI":"10.1145\/2645710.2645744"},{"key":"e_1_3_2_2_76_1","doi-asserted-by":"publisher","DOI":"10.1145\/3085504.3085526"},{"key":"e_1_3_2_2_77_1","volume-title":"Beyond parity: Fairness objectives for collaborative filtering. Advances in neural information processing systems 30","author":"Yao Sirui","year":"2017","unstructured":"Sirui Yao and Bert Huang. 2017. Beyond parity: Fairness objectives for collaborative filtering. Advances in neural information processing systems 30 (2017)."},{"key":"e_1_3_2_2_78_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132847.3132938"},{"key":"e_1_3_2_2_79_1","doi-asserted-by":"publisher","DOI":"10.1145\/3366424.3380048"},{"key":"e_1_3_2_2_80_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3271795"}],"event":{"name":"WWW '23: The ACM Web Conference 2023","location":"Austin TX USA","acronym":"WWW '23","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"]},"container-title":["Proceedings of the ACM Web Conference 2023"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3543507.3583204","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3543507.3583204","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:09:18Z","timestamp":1750183758000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3543507.3583204"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,4,30]]},"references-count":80,"alternative-id":["10.1145\/3543507.3583204","10.1145\/3543507"],"URL":"https:\/\/doi.org\/10.1145\/3543507.3583204","relation":{},"subject":[],"published":{"date-parts":[[2023,4,30]]},"assertion":[{"value":"2023-04-30","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}