{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,5]],"date-time":"2026-06-05T04:56:18Z","timestamp":1780635378797,"version":"3.54.1"},"publisher-location":"New York, NY, USA","reference-count":88,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,3,27]],"date-time":"2023-03-27T00:00:00Z","timestamp":1679875200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,3,27]]},"DOI":"10.1145\/3581641.3584092","type":"proceedings-article","created":{"date-parts":[[2023,3,27]],"date-time":"2023-03-27T16:16:52Z","timestamp":1679933812000},"page":"681-694","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["Human-Centered Deferred Inference: Measuring User Interactions and Setting Deferral Criteria for Human-AI Teams"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6911-786X","authenticated-orcid":false,"given":"Stephan J","family":"Lemmer","sequence":"first","affiliation":[{"name":"Robotics, University of Michigan, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4447-7818","authenticated-orcid":false,"given":"Anhong","family":"Guo","sequence":"additional","affiliation":[{"name":"Computer Science and Engineering, University of Michigan, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6454-9594","authenticated-orcid":false,"given":"Jason J","family":"Corso","sequence":"additional","affiliation":[{"name":"Electrical Engineering and Computer Science, University of Michigan, United States and Robotics, University of Michigan, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,3,27]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.279"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v7i1.5285"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33012429"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445717"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1609\/icaps.v25i1.13700"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00437"},{"key":"e_1_3_2_2_7_1","volume-title":"Proceedings of the 2010 Annual ACM Symposium on User Interface Software and Technology. ACM Press","author":"Bigham P","year":"2010","unstructured":"[7] Jeffrey\u00a0P Bigham, Chandrika Jayant, Hanjie Ji, Greg Little, Andrew Miller, Robert\u00a0C Miller, Robin Miller, Aubrey Tatarowicz, Brandyn White, Samual White, and Tom Yeh. 2010. VizWiz: nearly real-time answers to visual questions. In Proceedings of the 2010 Annual ACM Symposium on User Interface Software and Technology. ACM Press, New York, New York, USA, 333\u2013342."},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW.2010.5543821"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i5.20465"},{"key":"e_1_3_2_2_10_1","volume-title":"Proceedings of the 2016 IEEE International Symposium on Safety, Security, and Rescue Robotics. IEEE Press","author":"Cacace J.","unstructured":"[10] J. Cacace, A. Finzi, V. Lippiello, M. Furci, N. Mimmo, and L. Marconi. 2016. A control architecture for multiple drones operated via multimodal interaction in search & rescue mission. In Proceedings of the 2016 IEEE International Symposium on Safety, Security, and Rescue Robotics. IEEE Press, Lausanne, Switzerland, 233\u2013239."},{"key":"e_1_3_2_2_11_1","volume-title":"Proceedings of the 2019 Conference on Advances in Neural Information Processing Systems. Curran Associates","author":"Cadene Remi","year":"2019","unstructured":"[11] Remi Cadene and Corentin Dancette. 2019. RUBi: Reducing Unimodal Biases for Visual Question Answering. In Proceedings of the 2019 Conference on Advances in Neural Information Processing Systems. Curran Associates, Vancouver, British Columbia, Canada, 839\u2013850."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300234"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376402"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2788613"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/11774303_17"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445131"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300931"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58577-8_7"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.1970.1054406"},{"key":"e_1_3_2_2_20_1","volume-title":"Proceedings of the 2016 Conference on Advances in Neural Information Processing Systems. Curran Associates","author":"Cortes Corinna","year":"2016","unstructured":"[20] Corinna Cortes, Giulia DeSalvo, and Mehryar Mohri. 2016. Boosting with Abstention. In Proceedings of the 2016 Conference on Advances in Neural Information Processing Systems. Curran Associates, Barcelona, Spain, 1660\u20131668."},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3098279.3098539"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00721"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-45665-1_6"},{"key":"e_1_3_2_2_24_1","volume-title":"Proceedings of the 2016 International Conference on Machine Learning. PMLR","author":"Gal Yarin","year":"2016","unstructured":"[24] Yarin Gal and Zoubin Ghahramani. 2016. Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning. In Proceedings of the 2016 International Conference on Machine Learning. PMLR, New York, New York, USA, 1050\u20131059."},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICPR48806.2021.9412590"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1016\/0167-8655(92)90125-J"},{"key":"e_1_3_2_2_27_1","volume-title":"Proceedings of the 2017 Conference on Advances in Neural Information Processing Systems. Curran Associates","author":"Geifman Yonatan","year":"2017","unstructured":"[27] Yonatan Geifman and Ran El-Yaniv. 2017. Selective Classification for Deep Neural Networks. In Proceedings of the 2017 Conference on Advances in Neural Information Processing Systems. Curran Associates, Long Beach, California, USA, 4878\u20134887."},{"key":"e_1_3_2_2_28_1","volume-title":"Proceedings of the 2019 International Conference on Machine Learning. ACM Press","author":"Geifman Yonatan","year":"2019","unstructured":"[28] Yonatan Geifman and Ran El-Yaniv. 2019. SelectiveNet: A Deep Neural Network with an Integrated Reject Option. In Proceedings of the 2019 International Conference on Machine Learning. ACM Press, Long Beach, California, USA, 2151\u20132159."},{"key":"e_1_3_2_2_29_1","volume-title":"Proceedings of the 2017 International Conference on Machine Learning. PMLR","author":"Guo Chuan","year":"2017","unstructured":"[29] Chuan Guo, Geoff Pleiss, Yu Sun, and Kilian\u00a0Q. Weinberger. 2017. On Calibration of Modern Neural Networks. In Proceedings of the 2017 International Conference on Machine Learning. PMLR, Sydney, New South Wales, Australia, 1321\u20131330."},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025781"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00380"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICAICT.2014.7036000"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2806416.2806491"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2018.8460699"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01219-9_47"},{"key":"e_1_3_2_2_36_1","volume-title":"Real conversations with artificial intelligence: A comparison between human\u2013human online conversations and human\u2013chatbot conversations. Computers in Human Behavior 49 (Aug","author":"Hill Jennifer","year":"2015","unstructured":"[36] Jennifer Hill, W. Randolph\u00a0Ford, and Ingrid\u00a0G. Farreras. 2015. Real conversations with artificial intelligence: A comparison between human\u2013human online conversations and human\u2013chatbot conversations. Computers in Human Behavior 49 (Aug. 2015), 245\u2013250."},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174042"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v4i1.13288"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/D14-1086"},{"key":"e_1_3_2_2_40_1","first-page":"19","article-title":"A Care Robot with Ethical Sensing System for Older Adults at Home","volume":"22","author":"Kim Jong-Wook","year":"2022","unstructured":"[40] Jong-Wook Kim, Young-Lim Choi, Sang-Hyun Jeong, and Jeonghye Han. 2022. A Care Robot with Ethical Sensing System for Older Adults at Home. Sensors 22, 19 (Oct. 2022), 7515.","journal-title":"Sensors"},{"key":"e_1_3_2_2_41_1","volume-title":"Beam","author":"Kompa Benjamin","year":"2021","unstructured":"[41] Benjamin Kompa, Jasper Snoek, and Andrew\u00a0L. Beam. 2021. Second opinion needed: communicating uncertainty in medical machine learning. npj Digital Medicine 4, 1 (Dec. 2021), 4."},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2016.2516982"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501999"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501966"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-017-17876-z"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00074"},{"key":"e_1_3_2_2_47_1","volume-title":"Proceedings of the 2023 AAAI Conference on Artificial Intelligence","author":"Lemmer J.","unstructured":"[47] Stephan\u00a0J. Lemmer and Jason\u00a0J. Corso. 2023. Evaluating and Improving Interactions with Hazy Oracles. In Proceedings of the 2023 AAAI Conference on Artificial Intelligence. AAAI Press, Washington, District of Columbia, USA, 9."},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445181"},{"key":"e_1_3_2_2_49_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01234-2_34"},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10602-1_48"},{"key":"e_1_3_2_2_51_1","unstructured":"[51] Brian Lucena. 2018. Spline-Based Probability Calibration."},{"key":"e_1_3_2_2_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858288"},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025814"},{"key":"e_1_3_2_2_54_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D17-1097"},{"key":"e_1_3_2_2_55_1","volume-title":"Explicit Bias Discovery in Visual Question Answering Models. In 2019 IEEE\/CVF Conference on Computer Vision and Pattern Recognition. IEEE PRess","author":"Manjunatha Varun","year":"2019","unstructured":"[55] Varun Manjunatha, Nirat Saini, and Larry\u00a0S. Davis. 2019. Explicit Bias Discovery in Visual Question Answering Models. In 2019 IEEE\/CVF Conference on Computer Vision and Pattern Recognition. IEEE PRess, Long Beach, California, USA, 9554\u20139563."},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.9"},{"key":"e_1_3_2_2_57_1","volume-title":"Proceedings of the 2020 International Symposium on Experimental Robotics. Springer, La Valletta, Malta, 491\u2013501","author":"Mees Oier","year":"2020","unstructured":"[57] Oier Mees and Wolfram Burgard. 2020. Composing Pick-and-Place Tasks By Grounding Language. In Proceedings of the 2020 International Symposium on Experimental Robotics. Springer, La Valletta, Malta, 491\u2013501."},{"key":"e_1_3_2_2_58_1","unstructured":"[58] Azadeh\u00a0Sadat Mozafari Hugo\u00a0Siqueira Gomes Wilson Le\u00e3o Steeven Janny and Christian Gagn\u00e9. 2018. Attended Temperature Scaling: A Practical Approach for Calibrating Deep Neural Networks."},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"publisher","DOI":"10.1109\/RO-MAN50785.2021.9515551"},{"key":"e_1_3_2_2_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/3242587.3242666"},{"key":"e_1_3_2_2_61_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2019.8683351"},{"key":"e_1_3_2_2_62_1","volume-title":"Proceedings of the 2018 Conference on Robot Learning. PMLR","author":"Nyga Daniel","year":"2018","unstructured":"[62] Daniel Nyga, Subhro Roy, Rohan Paul, Daehyung Park, Mihai Pomarlan, Michael Beetz, and Nicholas Roy. 2018. Grounding Robot Plans from Natural Language Instructions with Incomplete World Knowledge. In Proceedings of the 2018 Conference on Robot Learning. PMLR, Zurich, Switzerland, 714\u2013723."},{"key":"e_1_3_2_2_63_1","unstructured":"[63] Amelia\u00a0Elizabeth Pollard and Jonathan\u00a0L. Shapiro. 2020. Visual Question Answering as a Multi-Task Problem."},{"key":"e_1_3_2_2_64_1","unstructured":"[64] Prakruthi Prabhakar Nitish Kulkarni and Linghao Zhang. 2018. Question Relevance in Visual Question Answering."},{"key":"e_1_3_2_2_65_1","volume-title":"Proceedings of the 2019 International Conference on Machine Learning. ACM Press","author":"Raghu Maithra","year":"2019","unstructured":"[65] Maithra Raghu, Katy Blumer, Rory Sayres, Ziad Obermeyer, Robert Kleinberg, Sendhil Mullainathan, and Jon Kleinberg. 2019. Direct Uncertainty Prediction for Medical Second Opinions. In Proceedings of the 2019 International Conference on Machine Learning. ACM Press, Long Beach, California, USA, 5281\u20135290."},{"key":"e_1_3_2_2_66_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1264"},{"key":"e_1_3_2_2_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/2960310.2960333"},{"key":"e_1_3_2_2_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/3192714.3192821"},{"key":"e_1_3_2_2_70_1","unstructured":"[70] Amir Rosenfeld Richard Zemel and John\u00a0K. Tsotsos. 2018. The Elephant in the Room."},{"key":"e_1_3_2_2_71_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W17-5536"},{"key":"e_1_3_2_2_72_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-019-01228-7"},{"key":"e_1_3_2_2_73_1","doi-asserted-by":"publisher","DOI":"10.15607\/RSS.2022.XVIII.065"},{"key":"e_1_3_2_2_74_1","doi-asserted-by":"publisher","DOI":"10.15607\/RSS.2018.XIV.028"},{"key":"e_1_3_2_2_75_1","volume-title":"Proceedings of the 2014 International Conference on Learning Representations. OpenReview","author":"Szegedy Christian","year":"2014","unstructured":"[75] Christian Szegedy, Wojciech Zaremba, Ilya Sutskever, Joan Bruna, Dumitru Erhan, Ian Goodfellow, and Rob Fergus. 2014. Intriguing properties of neural networks. In Proceedings of the 2014 International Conference on Learning Representations. OpenReview, Banff, Alberta, Canada, 10."},{"key":"e_1_3_2_2_76_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.177"},{"key":"e_1_3_2_2_77_1","volume-title":"What kinds of errors do reference resolution models make and what can we learn from them?. In 2022 Findings of the Association for Computational Linguistics","author":"S\u00e1nchez Jorge","year":"1971","unstructured":"[77] Jorge S\u00e1nchez, Mauricio Mazuecos, Hern\u00e1n Maina, and Luciana Benotti. 2022. What kinds of errors do reference resolution models make and what can we learn from them?. In 2022 Findings of the Association for Computational Linguistics. ACL Press, Seattle, Washington, USA, 1971\u20131986."},{"key":"e_1_3_2_2_78_1","volume-title":"Vision-and-Dialog Navigation. In Proceedings of the 2020 Conference on Robot Learning. PMLR, Virtual, 394\u2013406","author":"Thomason Jesse","year":"2020","unstructured":"[78] Jesse Thomason, Michael Murray, Maya Cakmak, and Luke Zettlemoyer. 2020. Vision-and-Dialog Navigation. In Proceedings of the 2020 Conference on Robot Learning. PMLR, Virtual, 394\u2013406."},{"key":"e_1_3_2_2_79_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW56347.2022.00514"},{"key":"e_1_3_2_2_80_1","doi-asserted-by":"publisher","DOI":"10.1145\/3394171.3413812"},{"key":"e_1_3_2_2_81_1","doi-asserted-by":"publisher","DOI":"10.1109\/SSP.2011.5967817"},{"key":"e_1_3_2_2_82_1","volume-title":"Proceedings of the 2019 Conference on Advances in Neural Information Processing Systems. Curran Associates","author":"Widmann David","year":"2019","unstructured":"[82] David Widmann, Fredrik Lindsten, and Dave Zachariah. 2019. Calibration tests in multi-class classification: A unifying framework. In Proceedings of the 2019 Conference on Advances in Neural Information Processing Systems. Curran Associates, Vancouver, British Columbia, Canada, 12236\u201312246."},{"key":"e_1_3_2_2_83_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501904"},{"key":"e_1_3_2_2_84_1","doi-asserted-by":"publisher","DOI":"10.1186\/s40648-019-0132-3"},{"key":"e_1_3_2_2_85_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3375709"},{"key":"e_1_3_2_2_86_1","doi-asserted-by":"publisher","DOI":"10.1145\/3125739.3125766"},{"key":"e_1_3_2_2_87_1","volume-title":"\u00a0E. Bellamy","author":"Zhang Yunfeng","year":"2020","unstructured":"[87] Yunfeng Zhang, Q.\u00a0Vera Liao, and Rachel K.\u00a0E. Bellamy. 2020. Effect of confidence and explanation on accuracy and trust calibration in AI-assisted decision making. In Proceedings of the 2020 Conference on Fairness, Accountability, and Transparency. ACM Press, Barcelona Spain, 295\u2013305."},{"key":"e_1_3_2_2_88_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3502036"}],"event":{"name":"IUI '23: 28th International Conference on Intelligent User Interfaces","location":"Sydney NSW Australia","acronym":"IUI '23","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 28th International Conference on Intelligent User Interfaces"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3581641.3584092","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3581641.3584092","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:36:21Z","timestamp":1750178181000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3581641.3584092"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,3,27]]},"references-count":88,"alternative-id":["10.1145\/3581641.3584092","10.1145\/3581641"],"URL":"https:\/\/doi.org\/10.1145\/3581641.3584092","relation":{},"subject":[],"published":{"date-parts":[[2023,3,27]]},"assertion":[{"value":"2023-03-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}