{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,5]],"date-time":"2026-06-05T04:44:58Z","timestamp":1780634698898,"version":"3.54.1"},"publisher-location":"New York, NY, USA","reference-count":47,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,7,18]],"date-time":"2023-07-18T00:00:00Z","timestamp":1689638400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,7,19]]},"DOI":"10.1145\/3539618.3591745","type":"proceedings-article","created":{"date-parts":[[2023,7,19]],"date-time":"2023-07-19T00:22:59Z","timestamp":1689726179000},"page":"18-27","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["On the Impact of Outlier Bias on User Clicks"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6284-1636","authenticated-orcid":false,"given":"Fatemeh","family":"Sarvi","sequence":"first","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9342-5272","authenticated-orcid":false,"given":"Ali","family":"Vardasbi","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9447-4172","authenticated-orcid":false,"given":"Mohammad","family":"Aliannejadi","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4722-5840","authenticated-orcid":false,"given":"Sebastian","family":"Schelter","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1086-0202","authenticated-orcid":false,"given":"Maarten","family":"de Rijke","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,7,18]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"crossref","unstructured":"Himan Abdollahpouri Robin Burke and Bamshad Mobasher. 2017. Controlling Popularity Bias in Learning-to-rank Recommendation. In RecSys. 42--46.","DOI":"10.1145\/3109859.3109912"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"crossref","unstructured":"Aman Agarwal Xuanhui Wang Cheng Li Michael Bendersky and Marc Najork. 2019a. Addressing Trust Bias for Unbiased Learning-to-rank. In WWW. 4--14.","DOI":"10.1145\/3308558.3313697"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"crossref","unstructured":"Aman Agarwal Ivan Zaitsev Xuanhui Wang Cheng Li Marc Najork and Thorsten Joachims. 2019b. Estimating Position Bias without Intrusive Interventions. In WSDM. 474--482.","DOI":"10.1145\/3289600.3291017"},{"key":"e_1_3_2_1_4_1","volume-title":"Is Font Size a Big Deal? A Transaction--Acquisition Utility Perspective on Comparative Price Promotions. Journal of Consumer Marketing","author":"Aggarwal Praveen","year":"2016","unstructured":"Praveen Aggarwal and Rajiv Vaidyanathan. 2016. Is Font Size a Big Deal? A Transaction--Acquisition Utility Perspective on Comparative Price Promotions. Journal of Consumer Marketing (2016)."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"crossref","unstructured":"Rakesh Agrawal Sreenivas Gollapudi Alan Halverson and Samuel Ieong. 2009. Diversifying Search Results. In WSDM. 5--14.","DOI":"10.1145\/1498759.1498766"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"Qingyao Ai Keping Bi Cheng Luo Jiafeng Guo and W Bruce Croft. 2018. Unbiased Learning to Rank with Unbiased Propensity Estimation. In SIGIR. 385--394.","DOI":"10.1145\/3209978.3209986"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3439861"},{"key":"e_1_3_2_1_8_1","volume-title":"Indian journal of psychological medicine","author":"Andrade Chittaranjan","year":"2018","unstructured":"Chittaranjan Andrade. 2018. Internal, External, and Ecological Validity in Research Design, Conduct, and Evaluation. Indian journal of psychological medicine, Vol. 40, 5 (2018), 498--499."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"crossref","unstructured":"Asia J Biega Krishna P Gummadi and Gerhard Weikum. 2018. Equity of Attention: Amortizing Individual Fairness in Rankings. In SIGIR. 405--414.","DOI":"10.1145\/3209978.3210063"},{"key":"e_1_3_2_1_10_1","first-page":"1","article-title":"Yahoo! Learning to Rank Challenge Overview","volume":"14","author":"Chapelle Olivier","year":"2011","unstructured":"Olivier Chapelle and Yi Chang. 2011. Yahoo! Learning to Rank Challenge Overview. Journal of Machine Learning Research, Vol. 14 (2011), 1--24.","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","unstructured":"Ruey-Cheng Chen Qingyao Ai Gaya Jayasinghe and W Bruce Croft. 2019. Correcting for Recency Bias in Job Recommendation. In CIKM. 2185--2188.","DOI":"10.1145\/3357384.3358131"},{"key":"e_1_3_2_1_12_1","volume-title":"Click Models for Web Search","author":"Chuklin Aleksandr","unstructured":"Aleksandr Chuklin, Ilya Markov, and Maarten de Rijke. 2015. Click Models for Web Search. Morgan & Claypool Publishers."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"Zhichong Fang Aman Agarwal and Thorsten Joachims. 2019. Intervention Harvesting for Context-dependent Examination-bias Estimation. In SIGIR. 825--834.","DOI":"10.1145\/3331184.3331238"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"crossref","unstructured":"Maria Heuss Fatemeh Sarvi and Maarten de Rijke. 2022. Fairness of Exposure in Light of Incomplete Exposure Estimation. In SIGIR. 759--769.","DOI":"10.1145\/3477495.3531977"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"Rolf Jagerman Harrie Oosterhuis and Maarten de Rijke. 2019. To Model or to Intervene: A Comparison of Counterfactual and Online Learning to Rank from User Interactions. In SIGIR. 15--24.","DOI":"10.1145\/3331184.3331269"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"Wen Jin Anthony K. H. Tung Jiawei Han and Wei Wang. 2006. Ranking Outliers Using Symmetric Neighborhood Relationship. In PAKDD. 577--593.","DOI":"10.1007\/11731139_68"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"crossref","unstructured":"Thorsten Joachims Laura Granka Bing Pan Helene Hembrooke and Geri Gay. 2005. Accurately Interpreting Clickthrough Data as Implicit Feedback. In SIGIR. 154--161.","DOI":"10.1145\/1076034.1076063"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/1229179.1229181"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Thorsten Joachims Adith Swaminathan and Tobias Schnabel. 2017. Unbiased Learning-to-rank with Biased Feedback. In WSDM. 781--789.","DOI":"10.1145\/3018661.3018699"},{"key":"e_1_3_2_1_20_1","volume-title":"Sally Rao Hill, and Indrit Troshani","author":"Kao Karen C","year":"2020","unstructured":"Karen C Kao, Sally Rao Hill, and Indrit Troshani. 2020. Effects of Cue Congruence and Perceived Cue Authenticity in Online Group Buying. Internet Research (2020)."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1207\/S15327078IN0204_03"},{"key":"e_1_3_2_1_22_1","volume-title":"COPOD: Copula-based Outlier Detection. In ICDM. 17--20.","author":"Li Zheng","year":"2020","unstructured":"Zheng Li, Yue Zhao, N Botta, C Ionescu, and X COPOD Hu. 2020. COPOD: Copula-based Outlier Detection. In ICDM. 17--20."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"crossref","unstructured":"Pavel Metrikov Fernando Diaz Sebastien Lahaie and Justin Rao. 2014. Whole Page Optimization: How Page Elements Interact with the Position Auction. In EC. 583--600.","DOI":"10.1145\/2600057.2602871"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"crossref","unstructured":"Marco Morik Ashudeep Singh Jessica Hong and Thorsten Joachims. 2020. Controlling Fairness and Bias in Dynamic Learning-to-rank. In SIGIR. 429--438.","DOI":"10.1145\/3397271.3401100"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"crossref","unstructured":"Harrie Oosterhuis and Maarten de Rijke. 2018. Differentiable Unbiased Online Learning to Rank. In CIKM. 1293--1302.","DOI":"10.1145\/3269206.3271686"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"crossref","unstructured":"Harrie Oosterhuis and Maarten de Rijke. 2020. Policy-aware Unbiased Learning to Rank for Top-k Rankings. In SIGIR. 489--498.","DOI":"10.1145\/3397271.3401102"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"crossref","unstructured":"Zohreh Ovaisi Ragib Ahsan Yifan Zhang Kathryn Vasilaky and Elena Zheleva. 2020. Correcting for Selection Bias in Learning-to-rank Systems. In WWW. 1863--1873.","DOI":"10.1145\/3366423.3380255"},{"key":"e_1_3_2_1_28_1","volume-title":"Context-Aware Learning to Rank with Self-Attention. ArXiv","author":"Pobrotyn Przemyslaw","year":"2020","unstructured":"Przemyslaw Pobrotyn, Tomasz Bartczak, Mikolaj Synowiec, Radoslaw Bialobrzeski, and Jaroslaw Bojar. 2020. Context-Aware Learning to Rank with Self-Attention. ArXiv, Vol. abs\/2005.10084 (2020)."},{"key":"e_1_3_2_1_29_1","unstructured":"Prolific. 2023. Data Annotation. https:\/\/www.prolific.co\/."},{"key":"e_1_3_2_1_30_1","volume-title":"arXiv preprint arXiv:1306.2597","author":"Qin Tao","year":"2013","unstructured":"Tao Qin and Tie-Yan Liu. 2013. Introducing LETOR 4.0 datasets. arXiv preprint arXiv:1306.2597 (2013)."},{"key":"e_1_3_2_1_31_1","unstructured":"Qualtrics. 2023. XM. https:\/\/www.qualtrics.com\/."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"crossref","unstructured":"Sridhar Ramaswamy Rajeev Rastogi and Kyuseok Shim. 2000. Efficient Algorithms for Mining Outliers from Large Data Sets. In SIGMOD. 427--438.","DOI":"10.1145\/335191.335437"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1080\/00401706.1999.10485670"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"crossref","unstructured":"Fatemeh Sarvi Maria Heuss Mohammad Aliannejadi Sebastian Schelter and Maarten de Rijke. 2022. Understanding and Mitigating the Effect of Outliers in Fair Ranking. In WSDM. 861--869.","DOI":"10.1145\/3488560.3498441"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1162\/089976601750264965"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"crossref","unstructured":"Ashudeep Singh and Thorsten Joachims. 2018. Fairness of Exposure in Rankings. In KDD. 2219--2228.","DOI":"10.1145\/3219819.3220088"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"crossref","unstructured":"Ali Vardasbi Maarten de Rijke and Ilya Markov. 2021. Mixture-Based Correction for Position and Trust Bias in Counterfactual Learning to Rank. In CIKM. 1869--1878.","DOI":"10.1145\/3459637.3482275"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"crossref","unstructured":"Ali Vardasbi Harrie Oosterhuis and Maarten de Rijke. 2020. When Inverse Propensity Scoring does not Work: Affine Corrections for Unbiased Learning to Rank. In CIKM. 1475--1484.","DOI":"10.1145\/3340531.3412031"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3532045"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2019.2932769"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"crossref","unstructured":"Huazheng Wang Ramsey Langley Sonwoo Kim Eric McCord-Snook and Hongning Wang. 2018b. Efficient Exploration of Gradient Space for Online Learning to Rank. In SIGIR. 145--154.","DOI":"10.1145\/3209978.3210045"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"crossref","unstructured":"Xuanhui Wang Michael Bendersky Donald Metzler and Marc Najork. 2016. Learning to Rank with Selection Bias in Personal Search. In SIGIR. 115--124.","DOI":"10.1145\/2911451.2911537"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"crossref","unstructured":"Xuanhui Wang Nadav Golbandi Michael Bendersky Donald Metzler and Marc Najork. 2018a. Position Bias Estimation for Unbiased Learning to Rank in Personal Search. In WSDM. 610--618.","DOI":"10.1145\/3159652.3159732"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3437963.3441751"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"crossref","unstructured":"Yisong Yue and Thorsten Joachims. 2009. Interactively Optimizing Information Retrieval Systems as A Dueling Bandits Problem. In ICML. 1201--1208.","DOI":"10.1145\/1553374.1553527"},{"key":"e_1_3_2_1_46_1","unstructured":"Yisong Yue Rajan Patel and Hein Roehrig. 2010. Beyond Position Bias: Examining Result Attractiveness as a Source of Presentation Bias in Clickthrough Data. In WWW. 1011--1018."},{"key":"e_1_3_2_1_47_1","volume-title":"LSCP: Locally Selective Combination in Parallel Outlier Ensembles. In ICDM. 585--593.","author":"Zhao Yue","year":"2019","unstructured":"Yue Zhao, Zain Nasrullah, Maciej K Hryniewicki, and Zheng Li. 2019. LSCP: Locally Selective Combination in Parallel Outlier Ensembles. In ICDM. 585--593."}],"event":{"name":"SIGIR '23: The 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Taipei Taiwan","acronym":"SIGIR '23","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591745","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3539618.3591745","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:47:01Z","timestamp":1750178821000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591745"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7,18]]},"references-count":47,"alternative-id":["10.1145\/3539618.3591745","10.1145\/3539618"],"URL":"https:\/\/doi.org\/10.1145\/3539618.3591745","relation":{},"subject":[],"published":{"date-parts":[[2023,7,18]]},"assertion":[{"value":"2023-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}