{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T05:04:00Z","timestamp":1750309440649,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":78,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,3,24]],"date-time":"2025-03-24T00:00:00Z","timestamp":1742774400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,3,24]]},"DOI":"10.1145\/3708359.3712092","type":"proceedings-article","created":{"date-parts":[[2025,3,19]],"date-time":"2025-03-19T12:50:34Z","timestamp":1742388634000},"page":"1093-1106","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Orbit: A Framework for Designing and Evaluating Multi-objective Rankers"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5016-7296","authenticated-orcid":false,"given":"Chenyang","family":"Yang","sequence":"first","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, Pennsylvania, USA,"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-6519-493X","authenticated-orcid":false,"given":"Tesi","family":"Xiao","sequence":"additional","affiliation":[{"name":"Amazon, Palo Alto, California, USA,"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-2385-1294","authenticated-orcid":false,"given":"Michael","family":"Shavlovsky","sequence":"additional","affiliation":[{"name":"Amazon, Palo Alto, California, USA,"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4450-4572","authenticated-orcid":false,"given":"Christian","family":"K\u00e4stner","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, Pennsylvania, USA,"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1630-0588","authenticated-orcid":false,"given":"Tongshuang","family":"Wu","sequence":"additional","affiliation":[{"name":"Human-Computer Interaction Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA,"}]}],"member":"320","published-online":{"date-parts":[[2025,3,24]]},"reference":[{"key":"e_1_3_3_3_2_2","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331202"},{"key":"e_1_3_3_3_3_2","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3274274"},{"key":"e_1_3_3_3_4_2","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702509"},{"key":"e_1_3_3_3_5_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642016"},{"key":"e_1_3_3_3_6_2","unstructured":"David Baehrens Timon Schroeter Stefan Harmeling Motoaki Kawanabe Katja Hansen and Klaus-Robert M\u00fcller. 2010. How to explain individual classification decisions. The Journal of Machine Learning Research 11 (2010) 1803\u20131831."},{"key":"e_1_3_3_3_7_2","doi-asserted-by":"crossref","unstructured":"Eric\u00a0PS Baumer. 2017. Toward human-centered algorithm design. Big Data & Society 4 2 (2017) 2053951717718854.","DOI":"10.1177\/2053951717718854"},{"key":"e_1_3_3_3_8_2","volume-title":"Statistics for experimenters: design, innovation, and discovery","author":"Box George E.\u00a0P.","year":"2009","unstructured":"George E.\u00a0P. Box. 2009. Statistics for experimenters: design, innovation, and discovery. Wiley-Blackwell."},{"key":"e_1_3_3_3_9_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581268"},{"key":"e_1_3_3_3_10_2","doi-asserted-by":"publisher","DOI":"10.1145\/3366423.3380122"},{"key":"e_1_3_3_3_11_2","doi-asserted-by":"crossref","unstructured":"Jiawei Chen Hande Dong Xiang Wang Fuli Feng Meng Wang and Xiangnan He. 2023. Bias and debias in recommender system: A survey and future directions. ACM Transactions on Information Systems 41 3 (2023) 1\u201339.","DOI":"10.1145\/3564284"},{"key":"e_1_3_3_3_12_2","doi-asserted-by":"publisher","DOI":"10.1145\/3580305.3599796"},{"key":"e_1_3_3_3_13_2","unstructured":"Andrew Collins Dominika Tkaczyk Akiko Aizawa and Joeran Beel. 2018. A study of position bias in digital library recommender systems. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1802.06565 (2018)."},{"key":"e_1_3_3_3_14_2","doi-asserted-by":"publisher","DOI":"10.1145\/2959100.2959190"},{"key":"e_1_3_3_3_15_2","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010139"},{"key":"e_1_3_3_3_16_2","unstructured":"Kaustubh\u00a0D. Dhole et\u00a0al. 2021. NL-Augmenter: A Framework for Task-Sensitive Natural Language Augmentation."},{"key":"e_1_3_3_3_17_2","doi-asserted-by":"publisher","DOI":"10.1145\/1718487.1718490"},{"key":"e_1_3_3_3_18_2","doi-asserted-by":"publisher","DOI":"10.1145\/604045.604056"},{"key":"e_1_3_3_3_19_2","doi-asserted-by":"crossref","unstructured":"Marco Ferrante Nicola Ferro and Norbert Fuhr. 2021. Towards meaningful statements in IR evaluation: Mapping evaluation measures to interval scales. IEEE Access 9 (2021) 136182\u2013136216.","DOI":"10.1109\/ACCESS.2021.3116857"},{"key":"e_1_3_3_3_20_2","doi-asserted-by":"crossref","unstructured":"Marco Ferrante Nicola Ferro and Eleonora Losiouk. 2020. How do interval scales help us with better understanding IR evaluation measures? Information Retrieval Journal 23 (2020) 289\u2013317.","DOI":"10.1007\/s10791-019-09362-z"},{"key":"e_1_3_3_3_21_2","doi-asserted-by":"publisher","DOI":"10.1145\/3121050.3121058"},{"key":"e_1_3_3_3_22_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-demos.6"},{"key":"e_1_3_3_3_23_2","doi-asserted-by":"publisher","DOI":"10.1007\/s10710-017-9314-z"},{"key":"e_1_3_3_3_24_2","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445423"},{"key":"e_1_3_3_3_25_2","doi-asserted-by":"publisher","DOI":"10.1145\/3616855.3636451"},{"key":"e_1_3_3_3_26_2","doi-asserted-by":"crossref","unstructured":"Florian Heimerl Steffen Koch Harald Bosch and Thomas Ertl. 2012. Visual classifier training for text document retrieval. IEEE Transactions on Visualization and Computer Graphics 18 12 (2012) 2839\u20132848.","DOI":"10.1109\/TVCG.2012.277"},{"key":"e_1_3_3_3_27_2","volume-title":"Contextual design: defining customer-centered systems","author":"Holtzblatt Karen","year":"1997","unstructured":"Karen Holtzblatt and Hugh Beyer. 1997. Contextual design: defining customer-centered systems. Morgan Kaufmann."},{"key":"e_1_3_3_3_28_2","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3271785"},{"key":"e_1_3_3_3_29_2","doi-asserted-by":"crossref","unstructured":"Kalervo J\u00e4rvelin and Jaana Kek\u00e4l\u00e4inen. 2002. Cumulated gain-based evaluation of IR techniques. ACM Transactions on Information Systems (TOIS) 20 4 (2002) 422\u2013446.","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_3_3_30_2","doi-asserted-by":"publisher","DOI":"10.1145\/3018661.3018699"},{"key":"e_1_3_3_3_31_2","doi-asserted-by":"crossref","unstructured":"Marius Kaminskas and Derek Bridge. 2016. Diversity serendipity novelty and coverage: a survey and empirical analysis of beyond-accuracy objectives in recommender systems. ACM Transactions on Interactive Intelligent Systems (TiiS) 7 1 (2016) 1\u201342.","DOI":"10.1145\/2926720"},{"key":"e_1_3_3_3_32_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642216"},{"key":"e_1_3_3_3_33_2","doi-asserted-by":"crossref","unstructured":"Anastasiia Klimashevskaia Dietmar Jannach Mehdi Elahi and Christoph Trattner. 2024. A survey on popularity bias in recommender systems. User Modeling and User-Adapted Interaction (2024) 1\u201358.","DOI":"10.1007\/s11257-024-09406-0"},{"key":"e_1_3_3_3_34_2","volume-title":"Proceedings of 16th European conference on computer-supported cooperative work-exploratory papers","author":"Koch Michael","year":"2018","unstructured":"Michael Koch, Kai von Luck, Jan Schwarzer, and Susanne Draheim. 2018. The novelty effect in large display deployments\u2013Experiences and lessons-learned for evaluating prototypes. In Proceedings of 16th European conference on computer-supported cooperative work-exploratory papers. European Society for Socially Embedded Technologies (EUSSET)."},{"key":"e_1_3_3_3_35_2","doi-asserted-by":"crossref","unstructured":"Josua Krause Adam Perer and Enrico Bertini. 2014. INFUSE: Interactive feature selection for predictive modeling of high dimensional data. IEEE transactions on visualization and computer graphics 20 12 (2014) 1614\u20131623.","DOI":"10.1109\/TVCG.2014.2346482"},{"key":"e_1_3_3_3_36_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581290"},{"key":"e_1_3_3_3_37_2","doi-asserted-by":"publisher","DOI":"10.1145\/3539597.3570395"},{"key":"e_1_3_3_3_38_2","doi-asserted-by":"publisher","DOI":"10.1145\/3580305.3599870"},{"key":"e_1_3_3_3_39_2","doi-asserted-by":"publisher","DOI":"10.1145\/3580305.3599482"},{"key":"e_1_3_3_3_40_2","unstructured":"Benjamin Marlin Richard\u00a0S Zemel Sam Roweis and Malcolm Slaney. 2012. Collaborative filtering and the missing at random assumption. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1206.5267 (2012)."},{"key":"e_1_3_3_3_41_2","doi-asserted-by":"crossref","unstructured":"Alistair Moffat. 2022. Batch evaluation metrics in information retrieval: Measures scales and meaning. IEEE Access 10 (2022) 105564\u2013105577.","DOI":"10.1109\/ACCESS.2022.3211668"},{"key":"e_1_3_3_3_42_2","doi-asserted-by":"publisher","DOI":"10.1145\/3366424.3382723"},{"key":"e_1_3_3_3_43_2","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510209"},{"key":"e_1_3_3_3_44_2","unstructured":"Long Ouyang Jeffrey Wu Xu Jiang Diogo Almeida Carroll Wainwright Pamela Mishkin Chong Zhang Sandhini Agarwal Katarina Slama Alex Ray et\u00a0al. 2022. Training language models to follow instructions with human feedback. Advances in neural information processing systems 35 (2022) 27730\u201327744."},{"key":"e_1_3_3_3_45_2","volume-title":"The filter bubble: How the new personalized web is changing what we read and how we think","author":"Pariser Eli","year":"2011","unstructured":"Eli Pariser. 2011. The filter bubble: How the new personalized web is changing what we read and how we think. Penguin."},{"key":"e_1_3_3_3_46_2","doi-asserted-by":"publisher","DOI":"10.1145\/3640543.3645144"},{"key":"e_1_3_3_3_47_2","doi-asserted-by":"publisher","DOI":"10.1093\/acprof:oso\/9780195173321.001.0001"},{"key":"e_1_3_3_3_48_2","unstructured":"Alexandre Rame Guillaume Couairon Corentin Dancette Jean-Baptiste Gaya Mustafa Shukor Laure Soulier and Matthieu Cord. 2024. Rewarded soups: towards pareto-optimal alignment by interpolating weights fine-tuned on diverse rewards. Advances in Neural Information Processing Systems 36 (2024)."},{"key":"e_1_3_3_3_49_2","unstructured":"Chandan\u00a0K. Reddy Llu\u00eds M\u00e0rquez Fran Valero Nikhil Rao Hugo Zaragoza Sambaran Bandyopadhyay Arnab Biswas Anlu Xing and Karthik Subbian. 2022. Shopping Queries Dataset: A Large-Scale ESCI Benchmark for Improving Product Search. (2022). arxiv:https:\/\/arXiv.org\/abs\/2206.06588"},{"key":"e_1_3_3_3_50_2","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372879"},{"key":"e_1_3_3_3_51_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.230"},{"key":"e_1_3_3_3_52_2","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"e_1_3_3_3_53_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.442"},{"key":"e_1_3_3_3_54_2","doi-asserted-by":"crossref","unstructured":"Horst\u00a0WJ Rittel and Melvin\u00a0M Webber. 1973. Dilemmas in a general theory of planning. Policy sciences 4 2 (1973) 155\u2013169.","DOI":"10.1007\/BF01405730"},{"key":"e_1_3_3_3_55_2","doi-asserted-by":"publisher","DOI":"10.1145\/2365952.2365961"},{"key":"e_1_3_3_3_56_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.468"},{"key":"e_1_3_3_3_57_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2015.24"},{"key":"e_1_3_3_3_58_2","doi-asserted-by":"publisher","unstructured":"Susan\u00a0Leigh Star and James\u00a0R. Griesemer. 1989. Institutional Ecology \u2018Translations\u2019 and Boundary Objects: Amateurs and Professionals in Berkeley\u2019s Museum of Vertebrate Zoology 1907-39. Social Studies of Science 19 3 (1989) 387\u2013420. 10.1177\/030631289019003001 arXiv:10.1177\/030631289019003001","DOI":"10.1177\/030631289019003001"},{"key":"e_1_3_3_3_59_2","doi-asserted-by":"publisher","DOI":"10.1145\/3240323.3240350"},{"key":"e_1_3_3_3_60_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581482"},{"key":"e_1_3_3_3_61_2","doi-asserted-by":"publisher","DOI":"10.1145\/3637528.3671597"},{"key":"e_1_3_3_3_62_2","unstructured":"Rachel Thomas and David Uminsky. 2020. The problem with metrics is a fundamental problem for AI. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2002.08512 (2020)."},{"key":"e_1_3_3_3_63_2","volume-title":"DELOS","author":"Toms Elaine\u00a0G","year":"2000","unstructured":"Elaine\u00a0G Toms et\u00a0al. 2000. Serendipitous Information Retrieval.. In DELOS. Citeseer."},{"key":"e_1_3_3_3_64_2","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462962"},{"key":"e_1_3_3_3_65_2","unstructured":"Yuyan Wang Cheenar Banerjee Samer Chucri Fabio Soldo Sriraj Badam Ed\u00a0H Chi and Minmin Chen. 2024. Diversifying by Intent in Recommender Systems. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2405.12327 (2024)."},{"key":"e_1_3_3_3_66_2","doi-asserted-by":"crossref","unstructured":"Haolun Wu Chen Ma Bhaskar Mitra Fernando Diaz and Xue Liu. 2022. A multi-objective optimization framework for multi-stakeholder fairness-aware recommendation. ACM Transactions on Information Systems 41 2 (2022) 1\u201329.","DOI":"10.1145\/3564285"},{"key":"e_1_3_3_3_67_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1073"},{"key":"e_1_3_3_3_68_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517582"},{"key":"e_1_3_3_3_69_2","doi-asserted-by":"crossref","unstructured":"Tongshuang Wu Daniel\u00a0S Weld and Jeffrey Heer. 2019. Local decision pitfalls in interactive machine learning: An investigation into feature selection in sentiment analysis. ACM Transactions on Computer-Human Interaction (TOCHI) 26 4 (2019) 1\u201327.","DOI":"10.1145\/3319616"},{"key":"e_1_3_3_3_70_2","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376451"},{"key":"e_1_3_3_3_71_2","doi-asserted-by":"publisher","DOI":"10.1145\/3624918.3625325"},{"key":"e_1_3_3_3_72_2","doi-asserted-by":"crossref","unstructured":"Chenyang Yang Rishabh Rustogi Rachel Brower-Sinning Grace\u00a0A Lewis Christian K\u00e4stner and Tongshuang Wu. 2023. Beyond Testers\u2019 Biases: Guiding Model Testing with Knowledge Bases using LLMs. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2310.09668 (2023).","DOI":"10.18653\/v1\/2023.findings-emnlp.901"},{"key":"e_1_3_3_3_73_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581388"},{"key":"e_1_3_3_3_74_2","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"e_1_3_3_3_75_2","doi-asserted-by":"publisher","DOI":"10.1145\/3298689.3346997"},{"key":"e_1_3_3_3_76_2","doi-asserted-by":"crossref","unstructured":"Yong Zheng and David\u00a0Xuejun Wang. 2022. A survey of recommender systems with multi-objective optimization. Neurocomputing 474 (2022) 141\u2013153.","DOI":"10.1016\/j.neucom.2021.11.041"},{"key":"e_1_3_3_3_77_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.630"},{"key":"e_1_3_3_3_78_2","unstructured":"Mu Zhu. 2004. Recall precision and average precision. Department of Statistics and Actuarial Science University of Waterloo Waterloo 2 30 (2004) 6."},{"key":"e_1_3_3_3_79_2","unstructured":"Lixin Zou Haitao Mao Xiaokai Chu Jiliang Tang Wenwen Ye Shuaiqiang Wang and Dawei Yin. 2022. A large scale search dataset for unbiased learning to rank. Advances in Neural Information Processing Systems 35 (2022) 1127\u20131139."}],"event":{"name":"IUI '25: 30th International Conference on Intelligent User Interfaces","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGCHI ACM Special Interest Group on Computer-Human Interaction"],"location":"Cagliari Italy","acronym":"IUI '25"},"container-title":["Proceedings of the 30th International Conference on Intelligent User Interfaces"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3708359.3712092","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3708359.3712092","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:09:46Z","timestamp":1750295386000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3708359.3712092"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,3,24]]},"references-count":78,"alternative-id":["10.1145\/3708359.3712092","10.1145\/3708359"],"URL":"https:\/\/doi.org\/10.1145\/3708359.3712092","relation":{},"subject":[],"published":{"date-parts":[[2025,3,24]]},"assertion":[{"value":"2025-03-24","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}