{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,23]],"date-time":"2026-04-23T05:49:11Z","timestamp":1776923351858,"version":"3.51.2"},"publisher-location":"New York, NY, USA","reference-count":71,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,6,12]],"date-time":"2023-06-12T00:00:00Z","timestamp":1686528000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["1900644,1927245"],"award-info":[{"award-number":["1900644,1927245"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,6,12]]},"DOI":"10.1145\/3593013.3594111","type":"proceedings-article","created":{"date-parts":[[2023,6,12]],"date-time":"2023-06-12T14:40:46Z","timestamp":1686580846000},"page":"1723-1734","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":21,"title":["Capturing Humans\u2019 Mental Models of AI: An Item Response Theory Approach"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9033-675X","authenticated-orcid":false,"given":"Markelle","family":"Kelly","sequence":"first","affiliation":[{"name":"University of California, Irvine, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9502-013X","authenticated-orcid":false,"given":"Aakriti","family":"Kumar","sequence":"additional","affiliation":[{"name":"University of California, Irvine, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9971-8378","authenticated-orcid":false,"given":"Padhraic","family":"Smyth","sequence":"additional","affiliation":[{"name":"University of California, Irvine, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1466-5647","authenticated-orcid":false,"given":"Mark","family":"Steyvers","sequence":"additional","affiliation":[{"name":"University of California, Irvine, USA"}]}],"member":"320","published-online":{"date-parts":[[2023,6,12]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1207\/s15324818ame0704_1"},{"key":"e_1_3_2_2_2_1","volume-title":"Burachas","author":"Alipour Kamran","year":"2021","unstructured":"Kamran Alipour, Arijit Ray, Xiao Lin, Michael Cogswell, J\u00fcrgen P. Schulze, Yi Yao, and Giedrius T. Burachas. 2021. Improving users\u2019 mental model with attention-directed counterfactual edits. (2021). arXiv:2110.06863"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1080\/02699939508409006"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2700832"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i13.17359"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v7i1.5285"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33012429"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445717"},{"key":"e_1_3_2_2_9_1","volume-title":"The nature of explanation. Nature 153, 3890","author":"Barnes Winston H F","year":"1944","unstructured":"Winston H F Barnes. 1944. The nature of explanation. Nature 153, 3890 (1944), 605\u2013605."},{"key":"e_1_3_2_2_10_1","first-page":"142","article-title":"Item response theory: An introduction to latent trait models to test and item development","volume":"7","author":"Bichi Ado Abdu","year":"2018","unstructured":"Ado Abdu Bichi and Rohaya Talib. 2018. Item response theory: An introduction to latent trait models to test and item development. International Journal of Evaluation and Research in Education 7, 2 (2018), 142\u2013151.","journal-title":"International Journal of Evaluation and Research in Education"},{"key":"e_1_3_2_2_11_1","volume-title":"Proceedings of the 25th International Conference on AI and Statistics (AI-Stats","author":"Bordt Sebastian","year":"2022","unstructured":"Sebastian Bordt and Ulrike Von Luxburg. 2022. A bandit model for human-machine decision making with private information and opacity. In Proceedings of the 25th International Conference on AI and Statistics (AI-Stats 2022). 7300\u20137319."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1177\/1071181319631392"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.662"},{"key":"e_1_3_2_2_14_1","volume-title":"2020 IEEE International Conference on Human-Machine Systems (ICHMS). 1\u20136.","author":"Moritz","unstructured":"Moritz C. Buehler and Thomas H. Weisswange. 2020. Theory of mind based communication for human agent cooperation. In 2020 IEEE International Conference on Human-Machine Systems (ICHMS). 1\u20136."},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359206"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1177\/0022243719851788"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1093\/schbul\/sbac038"},{"key":"e_1_3_2_2_18_1","volume-title":"Jennifer Wortman Vaughan, and Gagan Bansal","author":"Chen Valerie","year":"2023","unstructured":"Valerie Chen, Q. Vera Liao, Jennifer Wortman Vaughan, and Gagan Bansal. 2023. Understanding the Role of Human Intuition on Reliance in Human-AI Decision-Making with Explanations. arXiv preprint arXiv:2301.07255 (2023)."},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501831"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376638"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533240"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533221"},{"key":"e_1_3_2_2_23_1","volume-title":"Littman","author":"Druce Jeff","year":"2021","unstructured":"Jeff Druce, James Niehaus, Vanessa Moody, David D. Jensen, and Michael L. Littman. 2021. Brittle AI, causal confusion, and bad mental models: challenges and successes in the XAI program. CoRR abs\/2106.05506 (2021). arXiv:2106.05506"},{"key":"e_1_3_2_2_24_1","unstructured":"John Dunlosky and Janet Metcalfe. 2008. Metacognition. Sage Publications."},{"key":"e_1_3_2_2_25_1","volume-title":"Advances in Experimental Social Psychology.","author":"Dunning David","unstructured":"David Dunning. 2011. The Dunning\u2013Kruger effect: On being ignorant of one\u2019s own ignorance. In Advances in Experimental Social Psychology. Vol. 44. 247\u2013296."},{"key":"e_1_3_2_2_26_1","volume-title":"Bayesian Item Response Modeling: Theory and Applications","author":"Fox Jean-Paul","unstructured":"Jean-Paul Fox. 2010. Bayesian Item Response Modeling: Theory and Applications. Springer, New York."},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.cub.2005.08.041"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376316"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.stueduc.2009.10.002"},{"key":"e_1_3_2_2_30_1","volume-title":"Proceedings of the Twenty-fifth Pacific Asia Conference on Information Systems,. 1\u201314","author":"Hemmer Patrick","year":"2021","unstructured":"Patrick Hemmer, Max Schemmer, Michael V\u00f6ssing, and Niklas K\u00fchl. 2021. Human-AI complementarity in hybrid intelligence systems: A structured literature review. In Proceedings of the Twenty-fifth Pacific Asia Conference on Information Systems,. 1\u201314."},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1002\/aaai.12058"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1016\/0160-2896(79)90009-6"},{"key":"e_1_3_2_2_33_1","volume-title":"Proceedings of the International Joint Conference on AI (IJCAI","author":"Kamar Ece","year":"2016","unstructured":"Ece Kamar. 2016. Directions in hybrid iIntelligence: Complementing AI systems with human intelligence.. In Proceedings of the International Joint Conference on AI (IJCAI 2016). 4070\u20134073."},{"key":"e_1_3_2_2_34_1","volume-title":"Proceedings of the AAMAS Conference","volume":"12","author":"Kamar Ece","year":"2012","unstructured":"Ece Kamar, Severin Hacker, and Eric Horvitz. 2012. Combining human and machine intelligence in large-scale crowdsourcing.. In Proceedings of the AAMAS Conference, Vol. 12. 467\u2013474."},{"key":"e_1_3_2_2_35_1","volume-title":"UnifiedQA: Crossing format boundaries with a single QA system. arXiv preprint arXiv:2005.00700","author":"Khashabi Daniel","year":"2020","unstructured":"Daniel Khashabi, Sewon Min, Tushar Khot, Ashish Sabharwal, Oyvind Tafjord, Peter Clark, and Hannaneh Hajishirzi. 2020. UnifiedQA: Crossing format boundaries with a single QA system. arXiv preprint arXiv:2005.00700 (2020)."},{"key":"e_1_3_2_2_36_1","volume-title":"Machel Reid, Yutaka Matsuo, and Yusuke Iwasawa.","author":"Kojima Takeshi","year":"2022","unstructured":"Takeshi Kojima, Shixiang Shane Gu, Machel Reid, Yutaka Matsuo, and Yusuke Iwasawa. 2022. Large language models are zero-shot reasoners. In Advances in Neural Information Processing Systems. 22199\u201322213."},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2207678"},{"key":"e_1_3_2_2_38_1","volume-title":"Differentiating mental models of self and others: a hierarchical framework for knowledge assessment. PsyArXiv","author":"Kumar Aakriti","year":"2023","unstructured":"Aakriti Kumar, Padhraic Smyth, and Mark Steyvers. 2023. Differentiating mental models of self and others: a hierarchical framework for knowledge assessment. PsyArXiv (2023)."},{"key":"e_1_3_2_2_39_1","volume-title":"Proceedings of the Sixth Workshop on Natural Language for Artificial Intelligence (NL4AI","author":"Barbera David La","year":"2022","unstructured":"David La Barbera, Kevin Roitero, and Stefano Mizzaro. 2022. A hybrid human-in-the-loop framework for fact checking. In Proceedings of the Sixth Workshop on Natural Language for Artificial Intelligence (NL4AI 2022)."},{"key":"e_1_3_2_2_40_1","volume-title":"Towards a science of human-AI decision making: A survey of empirical studies. CoRR abs\/2112.11471","author":"Lai Vivian","year":"2021","unstructured":"Vivian Lai, Chacha Chen, Q. Vera Liao, Alison Smith-Renner, and Chenhao Tan. 2021. Towards a science of human-AI decision making: A survey of empirical studies. CoRR abs\/2112.11471 (2021). https:\/\/arxiv.org\/abs\/2112.11471"},{"key":"e_1_3_2_2_41_1","volume-title":"Proceedings of the 2005 IEEE International Conference on Robotics and Automation. 2767\u20132772","author":"Lee Sau","year":"2005","unstructured":"Sau lai Lee, Ivy Yee man Lau, S. Kiesler, and Chi-Yue Chiu. 2005. Human mental models of humanoid robots. In Proceedings of the 2005 IEEE International Conference on Robotics and Automation. 2767\u20132772."},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1186\/s41235-022-00364-y"},{"key":"e_1_3_2_2_43_1","volume-title":"Metacognition: An Overview.","author":"Livingston Jennifer A","year":"2003","unstructured":"Jennifer A Livingston. 2003. Metacognition: An Overview."},{"key":"e_1_3_2_2_44_1","volume-title":"Theory of machine: When do people rely on algorithms?Harvard Business School working paper series# 17-086","author":"Logg Jennifer Marie","year":"2017","unstructured":"Jennifer Marie Logg. 2017. Theory of machine: When do people rely on algorithms?Harvard Business School working paper series# 17-086 (2017)."},{"key":"e_1_3_2_2_45_1","volume-title":"\u201ctheory of machine","author":"Logg Jennifer M","unstructured":"Jennifer M Logg. 2022. The psychology of big data: Developing a \u201ctheory of machine\u201d to examine perceptions of algorithms. In The Psychology of Technology: Social Science Research in the Age of Big Data, Sandra Matz (Ed.). American Psychological Association, 349\u2013378."},{"key":"e_1_3_2_2_46_1","first-page":"183","article-title":"Performances of LOO and WAIC as IRT model selection methods","volume":"59","author":"Luo Yong","year":"2017","unstructured":"Yong Luo and Khaleel Al-Harbi. 2017. Performances of LOO and WAIC as IRT model selection methods. Psychological Test and Assessment Modeling 59, 2 (2017), 183.","journal-title":"Psychological Test and Assessment Modeling"},{"key":"e_1_3_2_2_47_1","volume-title":"The influence of shared mental models on team process and performance. Journal of Applied Psychology 85 (04","author":"Mathieu John","year":"2000","unstructured":"John Mathieu, Tonia Heffner, Gerald Goodwin, Eduardo Salas, and Janis Cannon-Bowers. 2000. The influence of shared mental models on team process and performance. Journal of Applied Psychology 85 (04 2000), 273\u2013283."},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1186\/s12911-021-01703-7"},{"key":"e_1_3_2_2_49_1","volume-title":"Explanation in artificial intelligence: Insights from the social sciences. Artificial intelligence 267","author":"Miller Tim","year":"2019","unstructured":"Tim Miller. 2019. Explanation in artificial intelligence: Insights from the social sciences. Artificial intelligence 267 (2019), 1\u201338."},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.obhdp.2006.09.002"},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397481.3450639"},{"key":"e_1_3_2_2_52_1","first-page":"610","article-title":"The utility of explainable AI in ad hoc human-machine teaming","volume":"34","author":"Paleja Rohan","year":"2021","unstructured":"Rohan Paleja, Muyleng Ghuy, Nadun Ranawaka Arachchige, Reed Jensen, and Matthew Gombolay. 2021. The utility of explainable AI in ad hoc human-machine teaming. In Advances in Neural Information Processing Systems, Vol. 34. 610\u2013623.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_2_53_1","volume-title":"A unifying framework for combining complementary strengths of humans and ML toward better predictive decision-making. arXiv preprint arXiv:2204.10806","author":"Rastogi Charvi","year":"2022","unstructured":"Charvi Rastogi, Liu Leqi, Kenneth Holstein, and Hoda Heidari. 2022. A unifying framework for combining complementary strengths of humans and ML toward better predictive decision-making. arXiv preprint arXiv:2204.10806 (2022)."},{"key":"e_1_3_2_2_54_1","doi-asserted-by":"publisher","DOI":"10.1177\/0146621697211002"},{"key":"e_1_3_2_2_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3492832"},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.1177\/1555343416682891"},{"key":"e_1_3_2_2_57_1","doi-asserted-by":"publisher","DOI":"10.1177\/0013164406296977"},{"key":"e_1_3_2_2_58_1","volume-title":"Why is theory of mind important for referential communication?Current Psychology 37","author":"Sidera Francesc","year":"2018","unstructured":"Francesc Sidera, Georgina Perpi\u00f1\u00e0, J\u00e8ssica Serrano, and Carles Rostan. 2018. Why is theory of mind important for referential communication?Current Psychology 37 (2018), 82\u201397."},{"key":"e_1_3_2_2_59_1","volume-title":"Cognition in Action","author":"Smyth Mary M","unstructured":"Mary M Smyth, Alan F Collins, Peter E Morris, and Philip Levy. 1994. Cognition in Action (2nd ed.). Lawrence Erlbaum Associates.","edition":"2"},{"key":"e_1_3_2_2_60_1","doi-asserted-by":"publisher","DOI":"10.2307\/1412107"},{"key":"e_1_3_2_2_61_1","doi-asserted-by":"publisher","DOI":"10.31234\/osf.io"},{"key":"e_1_3_2_2_62_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2111547119"},{"key":"e_1_3_2_2_63_1","doi-asserted-by":"publisher","DOI":"10.1037\/pas0000597"},{"key":"e_1_3_2_2_64_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41591-020-0942-0"},{"key":"e_1_3_2_2_65_1","unstructured":"W.J. van der Linden and R.K. Hambleton. 2013. Handbook of Modern Item Response Theory. Springer New York."},{"key":"e_1_3_2_2_66_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11222-016-9696-4"},{"key":"e_1_3_2_2_67_1","volume-title":"Deep learning for identifying metastatic breast cancer. arXiv preprint arXiv:1606.05718","author":"Wang Dayong","year":"2016","unstructured":"Dayong Wang, Aditya Khosla, Rishab Gargeya, Humayun Irshad, and Andrew H Beck. 2016. Deep learning for identifying metastatic breast cancer. arXiv preprint arXiv:1606.05718 (2016)."},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445645"},{"key":"e_1_3_2_2_69_1","volume-title":"Collective intelligence in human-AI teams: A Bayesian theory of mind approach. ArXiv abs\/2208.11660","author":"Westby Samuel","year":"2022","unstructured":"Samuel Westby and Christoph Riedl. 2022. Collective intelligence in human-AI teams: A Bayesian theory of mind approach. ArXiv abs\/2208.11660 (2022)."},{"key":"e_1_3_2_2_70_1","doi-asserted-by":"publisher","DOI":"10.1080\/10510974.2020.1749683"},{"key":"e_1_3_2_2_71_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2020\/212"}],"event":{"name":"FAccT '23: the 2023 ACM Conference on Fairness, Accountability, and Transparency","location":"Chicago IL USA","acronym":"FAccT '23"},"container-title":["2023 ACM Conference on Fairness Accountability and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3593013.3594111","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3593013.3594111","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3593013.3594111","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:37:19Z","timestamp":1750178239000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3593013.3594111"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,6,12]]},"references-count":71,"alternative-id":["10.1145\/3593013.3594111","10.1145\/3593013"],"URL":"https:\/\/doi.org\/10.1145\/3593013.3594111","relation":{},"subject":[],"published":{"date-parts":[[2023,6,12]]},"assertion":[{"value":"2023-06-12","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}