{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,18]],"date-time":"2026-08-18T01:44:19Z","timestamp":1787017459415,"version":"build-2736575974"},"publisher-location":"New York, NY, USA","reference-count":66,"publisher":"ACM","funder":[{"name":"Bosch Centre for Artificial Intelligence"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,6,23]]},"DOI":"10.1145\/3715275.3732060","type":"proceedings-article","created":{"date-parts":[[2025,6,23]],"date-time":"2025-06-23T13:03:13Z","timestamp":1750683793000},"page":"938-950","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["On Calibration in Multi-Distribution Learning"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2340-0942","authenticated-orcid":false,"given":"Rajeev","family":"Verma","sequence":"first","affiliation":[{"name":"Informatics Institute, University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5437-4030","authenticated-orcid":false,"given":"Volker","family":"Fischer","sequence":"additional","affiliation":[{"name":"Bosch Center for Artificial Intelligence, Renningen, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-1188-612X","authenticated-orcid":false,"given":"Eric","family":"Nalisnick","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Johns Hopkins University, Baltimore, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,6,23]]},"reference":[{"key":"e_1_3_3_2_2_2","doi-asserted-by":"crossref","unstructured":"Carlo Acerbi and Dirk Tasche. 2002. On the coherence of Expected Shortfall.","DOI":"10.1016\/S0378-4266(02)00283-2"},{"key":"e_1_3_3_2_3_2","doi-asserted-by":"publisher","DOI":"10.5555\/AAI27544679"},{"key":"e_1_3_3_2_4_2","doi-asserted-by":"crossref","unstructured":"Philippe Artzner Freddy Delbaen Jean-Marc Eber and David Heath. 1999. Coherent Measures of Risk. Mathematical Finance (1999).","DOI":"10.1111\/1467-9965.00068"},{"key":"e_1_3_3_2_5_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-45167-9_41"},{"key":"e_1_3_3_2_6_2","doi-asserted-by":"crossref","unstructured":"Aharon Ben-Tal Dick Den\u00a0Hertog Anja De\u00a0Waegenaere Bertrand Melenberg and Gijs Rennen. 2013. Robust solutions of optimization problems affected by uncertain probabilities. Management Science (2013).","DOI":"10.1287\/mnsc.1120.1641"},{"key":"e_1_3_3_2_7_2","doi-asserted-by":"publisher","DOI":"10.1515\/9781400831050"},{"key":"e_1_3_3_2_8_2","unstructured":"Gilles Blanchard Aniket\u00a0Anand Deshmukh Urun Dogan Gyemin Lee and Clayton Scott. 2021. Domain generalization by marginal transfer learning. Journal of Machine Learning Research (2021)."},{"key":"e_1_3_3_2_9_2","unstructured":"Gilles Blanchard Gyemin Lee and Clayton Scott. 2011. Generalizing from several related classification tasks to a new unlabeled sample. Advances in Neural Information Processing Systems (2011)."},{"key":"e_1_3_3_2_10_2","unstructured":"Avrim Blum Nika Haghtalab Ariel\u00a0D Procaccia and Mingda Qiao. 2017. Collaborative PAC learning. Advances in Neural Information Processing Systems (2017)."},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i8.16838"},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"crossref","unstructured":"Lev\u00a0M Bregman. 1967. The relaxation method of finding the common point of convex sets and its application to the solution of problems in convex programming. USSR computational mathematics and mathematical physics (1967).","DOI":"10.1016\/0041-5553(67)90040-7"},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"crossref","unstructured":"Jochen Br\u00f6cker. 2009. Reliability sufficiency and the decomposition of proper scores. Quarterly Journal of the Royal Meteorological Society: A journal of the atmospheric sciences applied meteorology and physical oceanography (2009).","DOI":"10.1002\/qj.456"},{"key":"e_1_3_3_2_14_2","volume-title":"Conference on Fairness, Accountability and Transparency","author":"Buolamwini Joy","year":"2018","unstructured":"Joy Buolamwini and Timnit Gebru. 2018. Gender shades: Intersectional accuracy disparities in commercial gender classification. In Conference on Fairness, Accountability and Transparency."},{"key":"e_1_3_3_2_15_2","doi-asserted-by":"crossref","unstructured":"A.\u00a0Philip Dawid. 1982. The Well-Calibrated Bayesian. J. Amer. Statist. Assoc. (1982).","DOI":"10.2307\/2287720"},{"key":"e_1_3_3_2_16_2","doi-asserted-by":"crossref","unstructured":"Morris\u00a0H. Degroot and Stephen\u00a0E. Fienberg. 1983. The Comparison and Evaluation of Forecasters. The Statistician (1983).","DOI":"10.2307\/2987588"},{"key":"e_1_3_3_2_17_2","unstructured":"Rabanus Derr and Robert\u00a0C Williamson. 2024. Four Facets of Forecast Felicity: Calibration Predictiveness Randomness and Regret. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2401.14483 (2024)."},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"crossref","unstructured":"John\u00a0C Duchi Peter\u00a0W Glynn and Hongseok Namkoong. 2021. Statistics of robust optimization: A generalized empirical likelihood approach. Mathematics of Operations Research (2021).","DOI":"10.1287\/moor.2020.1085"},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"publisher","DOI":"10.1145\/3406325.3451064"},{"key":"e_1_3_3_2_20_2","unstructured":"Cian Eastwood Alexander Robey Shashank Singh Julius Von\u00a0K\u00fcgelgen Hamed Hassani George\u00a0J Pappas and Bernhard Sch\u00f6lkopf. 2022. Probable domain generalization via quantile risk minimization. Advances in Neural Information Processing Systems (2022)."},{"key":"e_1_3_3_2_21_2","volume-title":"International Joint Conference on Artificial Intelligence","author":"Elkan Charles","year":"2001","unstructured":"Charles Elkan. 2001. The foundations of cost-sensitive learning. In International Joint Conference on Artificial Intelligence."},{"key":"e_1_3_3_2_22_2","unstructured":"Christian Fr\u00f6hlich and Robert\u00a0C Williamson. 2024. Data Models With Two Manifestations of Imprecision. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2404.09741 (2024)."},{"key":"e_1_3_3_2_23_2","unstructured":"Christian Fr\u00f6hlich and Robert\u00a0C. Williamson. 2024. Risk Measures and Upper Probabilities: Coherence and Stratification. Journal of Machine Learning Research (2024)."},{"key":"e_1_3_3_2_24_2","unstructured":"Christian Fr\u00f6hlich and Robert\u00a0C Williamson. 2024. Risk measures and upper probabilities: Coherence and stratification. Journal of Machine Learning Research (2024)."},{"key":"e_1_3_3_2_25_2","unstructured":"Christian Fr\u00f6hlich and Robert\u00a0C Williamson. 2024. Scoring Rules and Calibration for Imprecise Probabilities. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2410.23001 (2024)."},{"key":"e_1_3_3_2_26_2","volume-title":"International Conference on Machine Learning","author":"Globus-Harris Ira","year":"2023","unstructured":"Ira Globus-Harris, Declan Harrison, Michael Kearns, Aaron Roth, and Jessica Sorrell. 2023. Multicalibration as boosting for regression. In International Conference on Machine Learning."},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"crossref","unstructured":"Tilmann Gneiting and Adrian\u00a0E Raftery. 2007. Strictly Proper Scoring Rules Prediction and Estimation. J. Amer. Statist. Assoc. (2007).","DOI":"10.1198\/016214506000001437"},{"key":"e_1_3_3_2_28_2","unstructured":"Parikshit Gopalan Adam\u00a0Tauman Kalai Omer Reingold Vatsal Sharan and Udi Wieder. 2021. Omnipredictors. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2109.05389 (2021)."},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-43585-5"},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"crossref","unstructured":"Peter Gr\u00fcnwald. 2018. Safe probability. Journal of Statistical Planning and Inference (2018).","DOI":"10.1016\/j.jspi.2017.09.014"},{"key":"e_1_3_3_2_31_2","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/4643.001.0001"},{"key":"e_1_3_3_2_32_2","doi-asserted-by":"crossref","unstructured":"Peter\u00a0D Gr\u00fcnwald and A\u00a0Philip Dawid. 2004. Game theory maximum entropy minimum discrepancy and robust Bayesian decision theory. (2004).","DOI":"10.1214\/009053604000000553"},{"key":"e_1_3_3_2_33_2","volume-title":"International Conference on Machine Learning","author":"Guo Chuan","year":"2017","unstructured":"Chuan Guo, Geoff Pleiss, Yu Sun, and Kilian\u00a0Q Weinberger. 2017. On calibration of modern neural networks. In International Conference on Machine Learning."},{"key":"e_1_3_3_2_34_2","volume-title":"International Conference on Machine Learning","author":"Guo Chuan","year":"2017","unstructured":"Chuan Guo, Geoff Pleiss, Yu Sun, and Kilian\u00a0Q. Weinberger. 2017. On Calibration of Modern Neural Networks. In International Conference on Machine Learning."},{"key":"e_1_3_3_2_35_2","unstructured":"Nika Haghtalab Michael Jordan and Eric Zhao. 2022. On-Demand Sampling: Learning Optimally from Multiple Distributions. Advances in Neural Information Processing Systems (2022)."},{"key":"e_1_3_3_2_36_2","doi-asserted-by":"crossref","unstructured":"Alan H\u00e1jek. 2007. The Reference Class Problem is Your Problem Too. Synthese (2007).","DOI":"10.1007\/s11229-006-9138-5"},{"key":"e_1_3_3_2_37_2","volume-title":"International Conference on Machine Learning","author":"Hashimoto Tatsunori","year":"2018","unstructured":"Tatsunori Hashimoto, Megha Srivastava, Hongseok Namkoong, and Percy Liang. 2018. Fairness without demographics in repeated loss minimization. In International Conference on Machine Learning."},{"key":"e_1_3_3_2_38_2","volume-title":"International Conference on Machine Learning","author":"Hebert-Johnson Ursula","year":"2018","unstructured":"Ursula Hebert-Johnson, Michael Kim, Omer Reingold, and Guy Rothblum. 2018. Multicalibration: Calibration for the (Computationally-Identifiable) Masses. In International Conference on Machine Learning."},{"key":"e_1_3_3_2_39_2","doi-asserted-by":"publisher","DOI":"10.1145\/3593013.3594068"},{"key":"e_1_3_3_2_40_2","doi-asserted-by":"publisher","DOI":"10.1145\/130385.130424"},{"key":"e_1_3_3_2_41_2","volume-title":"Conference on Learning Theory","author":"Kleinberg Bobby","year":"2023","unstructured":"Bobby Kleinberg, Renato\u00a0Paes Leme, Jon Schneider, and Yifeng Teng. 2023. U-calibration: Forecasting for an unknown agent. In Conference on Learning Theory."},{"key":"e_1_3_3_2_42_2","doi-asserted-by":"crossref","unstructured":"Meelis Kull and Peter Flach. 2015. Novel decompositions of proper scoring rules for classification: score adjustment as precursor to calibration.","DOI":"10.1007\/978-3-319-23528-8_5"},{"key":"e_1_3_3_2_43_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-4-431-67891-5_4"},{"key":"e_1_3_3_2_44_2","doi-asserted-by":"publisher","DOI":"10.5555\/971143"},{"key":"e_1_3_3_2_45_2","volume-title":"Advances in Neural Information Processing Systems","author":"Mansour Yishay","year":"2008","unstructured":"Yishay Mansour, Mehryar Mohri, and Afshin Rostamizadeh. 2008. Domain adaptation with multiple sources. In Advances in Neural Information Processing Systems."},{"key":"e_1_3_3_2_46_2","volume-title":"International Conference on Machine Learning","author":"Martinez Natalia","year":"2020","unstructured":"Natalia Martinez, Martin Bertran, and Guillermo Sapiro. 2020. Minimax Pareto Fairness: A Multi Objective Perspective. In International Conference on Machine Learning."},{"key":"e_1_3_3_2_47_2","volume-title":"International Conference on Artificial Intelligence and Statistics","author":"Mehta Ronak","year":"2023","unstructured":"Ronak Mehta, Vincent Roulet, Krishna Pillutla, Lang Liu, and Zaid Harchaoui. 2023. Stochastic Optimization for Spectral Risk Measures. In International Conference on Artificial Intelligence and Statistics."},{"key":"e_1_3_3_2_48_2","volume-title":"International Conference on Machine Learning","author":"Meng Si\u00a0Yi","year":"2023","unstructured":"Si\u00a0Yi Meng and Robert\u00a0M. Gower. 2023. A model-based method for minimizing CVaR and beyond. In International Conference on Machine Learning."},{"key":"e_1_3_3_2_49_2","volume-title":"International Conference on Machine Learning","author":"Mohri Mehryar","year":"2019","unstructured":"Mehryar Mohri, Gary Sivek, and Ananda\u00a0Theertha Suresh. 2019. Agnostic federated learning. In International Conference on Machine Learning."},{"key":"e_1_3_3_2_50_2","unstructured":"Krikamol Muandet. 2022. Impossibility of collective intelligence. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2206.02786 (2022)."},{"key":"e_1_3_3_2_51_2","volume-title":"Advances in Neural Information Processing Systems","author":"Nguyen Huy","year":"2018","unstructured":"Huy Nguyen and Lydia Zakynthinou. 2018. Improved algorithms for collaborative PAC learning. In Advances in Neural Information Processing Systems."},{"key":"e_1_3_3_2_52_2","unstructured":"Georgy Noarov and Aaron Roth. 2024. Calibration for Decision Making: A Principled Approach to Trustworthy ML. Blog post."},{"key":"e_1_3_3_2_53_2","doi-asserted-by":"crossref","unstructured":"Ziad Obermeyer Brian Powers Christine Vogeli and Sendhil Mullainathan. 2019. Dissecting racial bias in an algorithm used to manage the health of populations. Science (2019).","DOI":"10.1126\/science.aax2342"},{"key":"e_1_3_3_2_54_2","doi-asserted-by":"crossref","unstructured":"Evgeni\u00a0Y Ovcharov. 2018. Proper scoring rules and Bregman divergence. (2018).","DOI":"10.3150\/16-BEJ857"},{"key":"e_1_3_3_2_55_2","doi-asserted-by":"crossref","unstructured":"Hamed Rahimian and Sanjay Mehrotra. 2022. Frameworks and Results in Distributionally Robust Optimization. Open Journal of Mathematical Optimization (2022).","DOI":"10.5802\/ojmo.15"},{"key":"e_1_3_3_2_56_2","unstructured":"Alfr\u00e9d R\u00e9nyi. 1961. On Measures of Entropy and Information."},{"key":"e_1_3_3_2_57_2","doi-asserted-by":"crossref","unstructured":"RT Rockafellar. 1970. Convex Analysis. Princeton Mathematical Series (1970).","DOI":"10.1515\/9781400873173"},{"key":"e_1_3_3_2_58_2","doi-asserted-by":"crossref","unstructured":"Terry Rockafellar. 2007. Coherent Approaches to Risk in Optimization Under Uncertainty.","DOI":"10.1287\/educ.1073.0032"},{"key":"e_1_3_3_2_59_2","volume-title":"International Conference on Machine Learning","author":"Rothblum Guy\u00a0N","year":"2021","unstructured":"Guy\u00a0N Rothblum and Gal Yona. 2021. Multi-group agnostic PAC learnability. In International Conference on Machine Learning."},{"key":"e_1_3_3_2_60_2","volume-title":"International Conference on Learning Representations","author":"Sagawa* Shiori","year":"2020","unstructured":"Shiori Sagawa*, Pang\u00a0Wei Koh*, Tatsunori\u00a0B. Hashimoto, and Percy Liang. 2020. Distributionally Robust Neural Networks. In International Conference on Learning Representations."},{"key":"e_1_3_3_2_61_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4419-1153-7_1232"},{"key":"e_1_3_3_2_62_2","volume-title":"International Conference on Artificial Intelligence and Statistics","author":"Vaicenavicius Juozas","year":"2019","unstructured":"Juozas Vaicenavicius, David Widmann, Carl Andersson, Fredrik Lindsten, Jacob Roll, and Thomas Sch\u00f6n. 2019. Evaluating model calibration in classification. In International Conference on Artificial Intelligence and Statistics."},{"key":"e_1_3_3_2_63_2","doi-asserted-by":"crossref","unstructured":"Leslie\u00a0G Valiant. 1984. A theory of the learnable. Commun. ACM (1984).","DOI":"10.1145\/800057.808710"},{"key":"e_1_3_3_2_64_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-71050-9_6"},{"key":"e_1_3_3_2_65_2","volume-title":"Theory of Games and Economic Behavior (60th Anniversary Commemorative Edition)","author":"Neumann John von","year":"1944","unstructured":"John von Neumann, Oskar Morgenstern, and Ariel Rubinstein. 1944. Theory of Games and Economic Behavior (60th Anniversary Commemorative Edition). Princeton University Press."},{"key":"e_1_3_3_2_66_2","volume-title":"Conference on Learning Theory","author":"Zhang Zihan","year":"2024","unstructured":"Zihan Zhang, Wenhao Zhan, Yuxin Chen, Simon\u00a0S Du, and Jason\u00a0D Lee. 2024. Optimal multi-distribution learning. In Conference on Learning Theory."},{"key":"e_1_3_3_2_67_2","unstructured":"Shengjia Zhao Michael Kim Roshni Sahoo Tengyu Ma and Stefano Ermon. 2021. Calibrating predictions to decisions: A novel approach to multi-class calibration. Advances in Neural Information Processing Systems (2021)."}],"event":{"name":"FAccT '25: The 2025 ACM Conference on Fairness, Accountability, and Transparency","location":"Athens Greece","acronym":"FAccT '25"},"container-title":["Proceedings of the 2025 ACM Conference on Fairness, Accountability, and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3715275.3732060","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,24]],"date-time":"2025-06-24T07:16:43Z","timestamp":1750749403000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3715275.3732060"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,23]]},"references-count":66,"alternative-id":["10.1145\/3715275.3732060","10.1145\/3715275"],"URL":"https:\/\/doi.org\/10.1145\/3715275.3732060","relation":{},"subject":[],"published":{"date-parts":[[2025,6,23]]},"assertion":[{"value":"2025-06-23","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}