{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T22:30:59Z","timestamp":1776119459683,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":65,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,5,11]],"date-time":"2024-05-11T00:00:00Z","timestamp":1715385600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,5,11]]},"DOI":"10.1145\/3613904.3642446","type":"proceedings-article","created":{"date-parts":[[2024,5,11]],"date-time":"2024-05-11T08:38:25Z","timestamp":1715416705000},"page":"1-19","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["Evaluating the Utility of Conformal Prediction Sets for AI-Advised Image Labeling"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9825-1411","authenticated-orcid":false,"given":"Dongping","family":"Zhang","sequence":"first","affiliation":[{"name":"Technology and Social Behavior, Northwestern University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9079-2376","authenticated-orcid":false,"given":"Angelos","family":"Chatzimparmpas","sequence":"additional","affiliation":[{"name":"Computer Science, Northwestern University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1086-6735","authenticated-orcid":false,"given":"Negar","family":"Kamali","sequence":"additional","affiliation":[{"name":"Computer Science, Northwestern University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6826-3550","authenticated-orcid":false,"given":"Jessica","family":"Hullman","sequence":"additional","affiliation":[{"name":"Computer Science, Northwestern University, United States"}]}],"member":"320","published-online":{"date-parts":[[2024,5,11]]},"reference":[{"key":"e_1_3_3_3_1_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.geb.2017.11.009"},{"key":"e_1_3_3_3_2_1","volume-title":"Concrete problems in AI safety. arXiv preprint arXiv:1606.06565","author":"Amodei Dario","year":"2016","unstructured":"Dario Amodei, Chris Olah, Jacob Steinhardt, Paul\u00a0F. Christiano, John Schulman, and Dan Man\u00e9. 2016. Concrete problems in AI safety. arXiv preprint arXiv:1606.06565 (2016)."},{"key":"e_1_3_3_3_3_1","volume-title":"Uncertainty sets for image classifiers using conformal prediction. arXiv preprint arXiv:2009.14193","author":"Angelopoulos Anastasios","year":"2020","unstructured":"Anastasios Angelopoulos, Stephen Bates, Jitendra Malik, and Michael\u00a0I Jordan. 2020. Uncertainty sets for image classifiers using conformal prediction. arXiv preprint arXiv:2009.14193 (2020)."},{"key":"e_1_3_3_3_4_1","volume-title":"A gentle introduction to conformal prediction and distribution-free uncertainty quantification. arXiv preprint arXiv:2107.07511","author":"Angelopoulos N","year":"2021","unstructured":"Anastasios\u00a0N Angelopoulos and Stephen Bates. 2021. A gentle introduction to conformal prediction and distribution-free uncertainty quantification. arXiv preprint arXiv:2107.07511 (2021)."},{"key":"e_1_3_3_3_5_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2022\/341"},{"key":"e_1_3_3_3_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445717"},{"key":"e_1_3_3_3_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376718"},{"key":"e_1_3_3_3_8_1","doi-asserted-by":"publisher","DOI":"10.1177\/0018720813482327"},{"key":"e_1_3_3_3_9_1","doi-asserted-by":"publisher","DOI":"10.1162\/coli.2006.32.1.13"},{"key":"e_1_3_3_3_10_1","doi-asserted-by":"publisher","DOI":"10.1609\/icwsm.v14i1.7282"},{"key":"e_1_3_3_3_11_1","doi-asserted-by":"publisher","DOI":"10.1177\/0956797613504966"},{"key":"e_1_3_3_3_12_1","doi-asserted-by":"publisher","DOI":"10.1037\/0003-066X.60.2.170"},{"key":"e_1_3_3_3_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"e_1_3_3_3_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/SMC.2017.8122985"},{"key":"e_1_3_3_3_15_1","volume-title":"WordNet: An electronic lexical database","author":"Fellbaum Christiane","unstructured":"Christiane Fellbaum. 1998. WordNet: An electronic lexical database. MIT Press."},{"key":"e_1_3_3_3_16_1","volume-title":"Deep learning-based image recognition for autonomous driving. IATSS research 43, 4","author":"Fujiyoshi Hironobu","year":"2019","unstructured":"Hironobu Fujiyoshi, Tsubasa Hirakawa, and Takayoshi Yamashita. 2019. Deep learning-based image recognition for autonomous driving. IATSS research 43, 4 (2019), 244\u2013252."},{"key":"e_1_3_3_3_17_1","volume-title":"Bayesian workflow. arXiv preprint arXiv:2011.01808","author":"Gelman Andrew","year":"2020","unstructured":"Andrew Gelman, Aki Vehtari, Daniel Simpson, Charles\u00a0C Margossian, Bob Carpenter, Yuling Yao, Lauren Kennedy, Jonah Gabry, Paul-Christian B\u00fcrkner, and Martin Modr\u00e1k. 2020. Bayesian workflow. arXiv preprint arXiv:2011.01808 (2020)."},{"key":"e_1_3_3_3_18_1","volume-title":"Proceedings of the ICLR.","author":"Gendler Asaf","year":"2022","unstructured":"Asaf Gendler, Tsui-Wei Weng, Luca Daniel, and Yaniv Romano. 2022. Adversarially robust conformal prediction. In Proceedings of the ICLR."},{"key":"e_1_3_3_3_19_1","volume-title":"Conformal Prediction With Conditional Guarantees. arXiv preprint arXiv:2305.12616","author":"Gibbs Isaac","year":"2023","unstructured":"Isaac Gibbs, John\u00a0J Cherian, and Emmanuel\u00a0J Cand\u00e8s. 2023. Conformal Prediction With Conditional Guarantees. arXiv preprint arXiv:2305.12616 (2023)."},{"key":"e_1_3_3_3_20_1","volume-title":"Proceedings of the ICLR.","author":"Goodfellow J","year":"2015","unstructured":"Ian\u00a0J Goodfellow, Jonathon Shlens, and Christian Szegedy. 2015. Explaining and harnessing adversarial examples. Proceedings of the ICLR."},{"key":"e_1_3_3_3_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359152"},{"key":"e_1_3_3_3_22_1","volume-title":"International conference on machine learning. PMLR, International Conference on Machine Learning (ICML), 1321\u20131330","author":"Guo Chuan","year":"2017","unstructured":"Chuan Guo, Geoff Pleiss, Yu Sun, and Kilian\u00a0Q Weinberger. 2017. On calibration of modern neural networks. In International conference on machine learning. PMLR, International Conference on Machine Learning (ICML), 1321\u20131330."},{"key":"e_1_3_3_3_23_1","volume-title":"A Statistical Framework for Measuring AI Reliance. arXiv preprint arXiv:2401.15356","author":"Guo Ziyang","year":"2024","unstructured":"Ziyang Guo, Yifan Wu, Jason Hartline, and Jessica Hullman. 2024. A Statistical Framework for Measuring AI Reliance. arXiv preprint arXiv:2401.15356 (2024)."},{"key":"e_1_3_3_3_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_3_3_25_1","volume-title":"Human-AI Complementarity in Hybrid Intelligence Systems: A Structured Literature Review.PACIS","author":"Hemmer Patrick","year":"2021","unstructured":"Patrick Hemmer, Max Schemmer, Michael V\u00f6ssing, and Niklas K\u00fchl. 2021. Human-AI Complementarity in Hybrid Intelligence Systems: A Structured Literature Review.PACIS (2021), 78."},{"key":"e_1_3_3_3_26_1","volume-title":"Proceedings of the ICLR.","author":"Hendrycks Dan","year":"2019","unstructured":"Dan Hendrycks and Thomas Dietterich. 2019. Benchmarking Neural Network Robustness to Common Corruptions and Perturbations. In Proceedings of the ICLR."},{"key":"e_1_3_3_3_27_1","volume-title":"Conformal prediction with local weights: randomization enables local guarantees. arXiv preprint arXiv:2310.07850","author":"Hore Rohan","year":"2023","unstructured":"Rohan Hore and Rina\u00a0Foygel Barber. 2023. Conformal prediction with local weights: randomization enables local guarantees. arXiv preprint arXiv:2310.07850 (2023)."},{"key":"e_1_3_3_3_28_1","volume-title":"Decision Theoretic Foundations for Experiments Evaluating Human Decisions. arXiv preprint arXiv:2401.15106","author":"Hullman Jessica","year":"2024","unstructured":"Jessica Hullman, Alex Kale, and Jason Hartline. 2024. Decision Theoretic Foundations for Experiments Evaluating Human Decisions. arXiv preprint arXiv:2401.15106 (2024)."},{"key":"e_1_3_3_3_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2018.2864889"},{"key":"e_1_3_3_3_30_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0142444"},{"key":"e_1_3_3_3_31_1","doi-asserted-by":"publisher","DOI":"10.1037\/a0025185"},{"key":"e_1_3_3_3_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3576841.3585931"},{"key":"e_1_3_3_3_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858558"},{"key":"e_1_3_3_3_34_1","volume-title":"Learning multiple layers of features from tiny images. Master\u2019s Thesis","author":"Krizhevsky Alex","unstructured":"Alex Krizhevsky. 2009. Learning multiple layers of features from tiny images. Master\u2019s Thesis, University of Toronto (2009)."},{"key":"e_1_3_3_3_35_1","first-page":"1","article-title":"Human-AI Collaboration via Conditional Delegation: A Case Study of Content Moderation. In Proceedings of the 2022 ACM CHI Conference on Human Factors in Computing Systems. ACM","volume":"54","author":"Lai Vivian","year":"2022","unstructured":"Vivian Lai, Samuel Carton, Rajat Bhatnagar, Q.\u00a0Vera Liao, Yunfeng Zhang, and Chenhao Tan. 2022. Human-AI Collaboration via Conditional Delegation: A Case Study of Content Moderation. In Proceedings of the 2022 ACM CHI Conference on Human Factors in Computing Systems. ACM, Article 54, 1\u201318\u00a0pages.","journal-title":"Article"},{"key":"e_1_3_3_3_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287590"},{"key":"e_1_3_3_3_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445522"},{"key":"e_1_3_3_3_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3479552"},{"key":"e_1_3_3_3_39_1","volume-title":"Towards deep learning models resistant to adversarial attacks. arXiv preprint arXiv:1706.06083","author":"Madry Aleksander","year":"2017","unstructured":"Aleksander Madry, Aleksandar Makelov, Ludwig Schmidt, Dimitris Tsipras, and Adrian Vladu. 2017. Towards deep learning models resistant to adversarial attacks. arXiv preprint arXiv:1706.06083 (2017)."},{"key":"e_1_3_3_3_40_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1722389115"},{"key":"e_1_3_3_3_41_1","volume-title":"Benchmarking robustness in object detection: Autonomous driving when winter is coming. arXiv preprint arXiv:1907.07484","author":"Michaelis Claudio","year":"2019","unstructured":"Claudio Michaelis, Benjamin Mitzkus, Robert Geirhos, Evgenia Rusak, Oliver Bringmann, Alexander\u00a0S Ecker, Matthias Bethge, and Wieland Brendel. 2019. Benchmarking robustness in object detection: Autonomous driving when winter is coming. arXiv preprint arXiv:1907.07484 (2019)."},{"key":"e_1_3_3_3_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/219717.219748"},{"key":"e_1_3_3_3_43_1","volume-title":"Proceedings of the CVPR workshops, Vol.\u00a02. IEEE.","author":"Nixon Jeremy","year":"2019","unstructured":"Jeremy Nixon, Michael\u00a0W Dusenberry, Linchuan Zhang, Ghassen Jerfel, and Dustin Tran. 2019. Measuring calibration in deep learning. In Proceedings of the CVPR workshops, Vol.\u00a02. IEEE."},{"key":"e_1_3_3_3_44_1","volume-title":"Pervasive label errors in test sets destabilize machine learning benchmarks. arXiv preprint arXiv:2103.14749","author":"Northcutt G","year":"2021","unstructured":"Curtis\u00a0G Northcutt, Anish Athalye, and Jonas Mueller. 2021. Pervasive label errors in test sets destabilize machine learning benchmarks. arXiv preprint arXiv:2103.14749 (2021)."},{"key":"e_1_3_3_3_45_1","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-36755-1_29"},{"key":"e_1_3_3_3_46_1","volume-title":"PyTorch: An imperative style, high-performance deep learning library. Advances in Neural Information Processing Systems 32","author":"Adam Paszke","year":"2019","unstructured":"Adam Paszke 2019. PyTorch: An imperative style, high-performance deep learning library. Advances in Neural Information Processing Systems 32 (2019)."},{"key":"e_1_3_3_3_47_1","first-page":"61","article-title":"Probabilistic outputs for support vector machines and comparisons to regularized likelihood methods","volume":"10","author":"Platt John","year":"1999","unstructured":"John Platt. 1999. Probabilistic outputs for support vector machines and comparisons to regularized likelihood methods. Advances in Large Margin Classifiers 10, 3 (1999), 61\u201374.","journal-title":"Advances in Large Margin Classifiers"},{"key":"e_1_3_3_3_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3581641.3584033"},{"key":"e_1_3_3_3_49_1","volume-title":"Dataset shift in machine learning","author":"Quinonero-Candela Joaquin","unstructured":"Joaquin Quinonero-Candela, Masashi Sugiyama, Anton Schwaighofer, and Neil\u00a0D Lawrence. 2008. Dataset shift in machine learning. Mit Press."},{"key":"e_1_3_3_3_50_1","volume-title":"Deep learning for medical image processing: Overview, challenges and the future. Classification in BioApps: Automation of Decision Making","author":"Razzak Muhammad\u00a0Imran","year":"2018","unstructured":"Muhammad\u00a0Imran Razzak, Saeeda Naz, and Ahmad Zaib. 2018. Deep learning for medical image processing: Overview, challenges and the future. Classification in BioApps: Automation of Decision Making (2018), 323\u2013350."},{"key":"e_1_3_3_3_51_1","volume-title":"Proceedings of the 14th International Joint Conference on Artificial Intelligence -","volume":"453","author":"Resnik Philip","year":"1995","unstructured":"Philip Resnik. 1995. Using Information Content to Evaluate Semantic Similarity in a Taxonomy. In Proceedings of the 14th International Joint Conference on Artificial Intelligence - Volume 1(IJCAI\u201995). Morgan Kaufmann Publishers Inc., 448\u2013453."},{"key":"e_1_3_3_3_52_1","first-page":"3581","article-title":"Classification with valid and adaptive coverage","volume":"33","author":"Romano Yaniv","year":"2020","unstructured":"Yaniv Romano, Matteo Sesia, and Emmanuel Candes. 2020. Classification with valid and adaptive coverage. Advances in Neural Information Processing Systems 33 (2020), 3581\u20133591.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_3_3_53_1","doi-asserted-by":"publisher","DOI":"10.1080\/2573234X.2021.1908861"},{"key":"e_1_3_3_3_54_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0042769"},{"key":"e_1_3_3_3_55_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2111547119"},{"key":"e_1_3_3_3_56_1","volume-title":"Designing decision support systems using counterfactual prediction sets. arXiv preprint arXiv:2306.03928","author":"Straitouri Eleni","year":"2023","unstructured":"Eleni Straitouri and Manuel\u00a0Gomez Rodriguez. 2023. Designing decision support systems using counterfactual prediction sets. arXiv preprint arXiv:2306.03928 (2023)."},{"key":"e_1_3_3_3_57_1","volume-title":"Guidelines for evaluating and expressing the uncertainty of NIST measurement results. Vol.\u00a01297","author":"Taylor N","unstructured":"Barry\u00a0N Taylor and Chris\u00a0E Kuyatt. 1994. Guidelines for evaluating and expressing the uncertainty of NIST measurement results. Vol.\u00a01297. National Institute of Standards and Technology."},{"key":"e_1_3_3_3_58_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2011.5995347"},{"key":"e_1_3_3_3_59_1","volume-title":"Belief in the law of small numbers.Psychological bulletin 76, 2","author":"Tversky Amos","year":"1971","unstructured":"Amos Tversky and Daniel Kahneman. 1971. Belief in the law of small numbers.Psychological bulletin 76, 2 (1971), 105."},{"key":"e_1_3_3_3_60_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-021-00520-6"},{"key":"e_1_3_3_3_61_1","volume-title":"Algorithmic learning in a random world. Vol.\u00a029","author":"Vovk Vladimir","unstructured":"Vladimir Vovk, Alexander Gammerman, and Glenn Shafer. 2005. Algorithmic learning in a random world. Vol.\u00a029. Springer."},{"key":"e_1_3_3_3_62_1","doi-asserted-by":"publisher","DOI":"10.5244\/C.30.87"},{"key":"e_1_3_3_3_63_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2021.3114679"},{"key":"e_1_3_3_3_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372852"},{"key":"e_1_3_3_3_65_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-68059-0_2"}],"event":{"name":"CHI '24: CHI Conference on Human Factors in Computing Systems","location":"Honolulu HI USA","acronym":"CHI '24","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction","SIGACCESS ACM Special Interest Group on Accessible Computing"]},"container-title":["Proceedings of the CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3613904.3642446","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3613904.3642446","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:56:48Z","timestamp":1750291008000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3613904.3642446"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5,11]]},"references-count":65,"alternative-id":["10.1145\/3613904.3642446","10.1145\/3613904"],"URL":"https:\/\/doi.org\/10.1145\/3613904.3642446","relation":{},"subject":[],"published":{"date-parts":[[2024,5,11]]},"assertion":[{"value":"2024-05-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}