{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,19]],"date-time":"2026-10-19T13:14:23Z","timestamp":1792415663035,"version":"4.4.0"},"reference-count":102,"publisher":"MIT Press","license":[{"start":{"date-parts":[[2026,9,4]],"date-time":"2026-09-04T00:00:00Z","timestamp":1788480000000},"content-version":"vor","delay-in-days":246,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"Schmidt AI 2050"},{"name":"ONR"},{"DOI":"10.13039\/100020895","name":"MIT-IBM Watson AI Lab","doi-asserted-by":"crossref","id":[{"id":"10.13039\/100020895","id-type":"DOI","asserted-by":"crossref"}]},{"name":"Reid Hoffman and the Siegel Family Foundation"},{"name":"King\u2019s College Cambridge"},{"DOI":"10.13039\/501100003343","name":"Cambridge Trust","doi-asserted-by":"crossref","id":[{"id":"10.13039\/501100003343","id-type":"DOI","asserted-by":"crossref"}]},{"name":"Turing AI Fellowship","award":["EP\/V025279\/1"],"award-info":[{"award-number":["EP\/V025279\/1"]}]},{"name":"Turing AI Fellowship","award":["EP\/V056522\/1"],"award-info":[{"award-number":["EP\/V056522\/1"]}]},{"name":"Turing AI Fellowship","award":["EP\/V056883\/1"],"award-info":[{"award-number":["EP\/V056883\/1"]}]},{"name":"Leverhulme Trust via CFI"}],"content-domain":{"domain":["direct.mit.edu"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,8,21]]},"abstract":"<jats:title>Abstract<\/jats:title>\n                  <jats:p>Recent advances in Artificial Intelligence (AI) have yielded powerful computational models that, by learning from vast amounts of human-generated data, are increasingly posited as approximate models of human cognition. However, we argue that many current evaluation paradigms for AI are insufficient for assessing human-like cognitive capabilities in these models. We identify a set of key shortcomings: a lack of human-validated labels, inadequate representation of human response variability and uncertainty, and reliance on simplified and ecologically invalid tasks. We support our claims by conducting a human evaluation study on nine existing AI benchmarks, suggesting major limitations in task and label designs. To address these limitations, we propose five concrete recommendations for future AI evaluation efforts that will enable more rigorous and meaningful understanding of human-like cognitive capacities in AI models.<\/jats:p>","DOI":"10.1162\/opmi.a.385","type":"journal-article","created":{"date-parts":[[2026,9,4]],"date-time":"2026-09-04T16:05:14Z","timestamp":1788537914000},"page":"1394-1412","update-policy":"https:\/\/doi.org\/10.1162\/mitpressjournals.corrections.policy","source":"Crossref","is-referenced-by-count":1,"title":["On Benchmarking Human-like Intelligence in Machines"],"prefix":"10.1162","volume":"10","author":[{"given":"Lance","family":"Ying","sequence":"first","affiliation":[{"name":"Department of Psychology, Harvard University, Cambridge, MA, USA"},{"name":"Department of Brain and Cognitive Sciences, Massachusetts Institute of Technology, Cambridge, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7032-716X","authenticated-orcid":true,"given":"Katherine M.","family":"Collins","sequence":"additional","affiliation":[{"name":"Department of Engineering, University of Cambridge, Cambridge, UK"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lionel","family":"Wong","sequence":"additional","affiliation":[{"name":"Department of Brain and Cognitive Sciences, Massachusetts Institute of Technology, Cambridge, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4121-7479","authenticated-orcid":true,"given":"Ilia","family":"Sucholutsky","sequence":"additional","affiliation":[{"name":"Department of Computer Science, New York University, New York, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ryan","family":"Liu","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Princeton University, Princeton, NJ, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1915-7158","authenticated-orcid":true,"given":"Adrian","family":"Weller","sequence":"additional","affiliation":[{"name":"Department of Engineering, University of Cambridge, Cambridge, UK"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5734-0653","authenticated-orcid":true,"given":"Tianmin","family":"Shu","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5138-7255","authenticated-orcid":true,"given":"Thomas L.","family":"Griffiths","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Princeton University, Princeton, NJ, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1925-2035","authenticated-orcid":true,"given":"Joshua B.","family":"Tenenbaum","sequence":"additional","affiliation":[{"name":"Department of Brain and Cognitive Sciences, Massachusetts Institute of Technology, Cambridge, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"281","published-online":{"date-parts":[[2026,8,21]]},"reference":[{"key":"2026090412050628100_bib1","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/1167.003.0002","volume-title":"Cognitive modeling and intelligent tutoring","author":"Anderson","year":"1990"},{"key":"2026090412050628100_bib2","author":"Ashokkumar","year":"2024","journal-title":"Predicting results of social science experiments using large language models"},{"key":"2026090412050628100_bib3","doi-asserted-by":"publisher","first-page":"0064","DOI":"10.1038\/s41562-017-0064","article-title":"Rational quantitative attribution of beliefs, desires and percepts in human mentalizing","volume":"1","author":"Baker","year":"2017","journal-title":"Nature Human Behaviour"},{"key":"2026090412050628100_bib4","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1146\/annurev.psych.52.1.1","article-title":"Social cognitive theory: An agentic perspective","volume":"52","author":"Bandura","year":"2001","journal-title":"Annual Review of Psychology"},{"key":"2026090412050628100_bib5","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2511.04703","article-title":"Measuring what matters: Construct validity in large language model benchmarks","author":"Bean","year":"2025","journal-title":"arXiv"},{"key":"2026090412050628100_bib6","doi-asserted-by":"publisher","first-page":"2905","DOI":"10.3389\/fpsyg.2019.02905","article-title":"Systematic review and inventory of theory of mind measures for young children","volume":"10","author":"Beaudoin","year":"2020","journal-title":"Frontiers in Psychology"},{"issue":"1","key":"2026090412050628100_bib7","doi-asserted-by":"publisher","first-page":"5","DOI":"10.2307\/2184371","article-title":"Psychologism and behaviorism","volume":"90","author":"Block","year":"1981","journal-title":"Philosophical Review"},{"issue":"1","key":"2026090412050628100_bib8","doi-asserted-by":"publisher","first-page":"B25","DOI":"10.1016\/S0010-0277(00)00096-2","article-title":"Two reasons to abandon the false belief task as a test of theory of mind","volume":"77","author":"Bloom","year":"2000","journal-title":"Cognition"},{"key":"2026090412050628100_bib9","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2405.13063","article-title":"Aurora: A foundation model of the atmosphere","author":"Bodnar","year":"2024","journal-title":"arXiv"},{"issue":"1756","key":"2026090412050628100_bib10","doi-asserted-by":"publisher","first-page":"20170280","DOI":"10.1098\/rstb.2017.0280","article-title":"Measuring and understanding individual differences in cognition","volume":"373","author":"Boogert","year":"2018","journal-title":"Philosophical Transactions of the Royal Society of London: Series B, Biological Sciences"},{"key":"2026090412050628100_bib11","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.4395751","volume-title":"Using LLMs for market research","author":"Brand","year":"2023"},{"key":"2026090412050628100_bib12","article-title":"On the utility of learning about humans for human-ai coordination","volume":"32","author":"Carroll","year":"2019","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2026090412050628100_bib13","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2502.20349","article-title":"Naturalistic computational cognitive science: Towards generalizable models and theories that capture the full range of natural behavior","author":"Carvalho","year":"2025","journal-title":"arXiv"},{"key":"2026090412050628100_bib14","doi-asserted-by":"publisher","DOI":"10.1093\/oso\/9780192865366.001.0001","volume-title":"Imagining AI: How the world sees intelligent machines","author":"Cave","year":"2023"},{"issue":"7","key":"2026090412050628100_bib15","doi-asserted-by":"publisher","first-page":"335","DOI":"10.1016\/j.tics.2006.05.006","article-title":"Probabilistic models of language processing and acquisition","volume":"10","author":"Chater","year":"2006","journal-title":"Trends in Cognitive Sciences"},{"key":"2026090412050628100_bib16","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2402.15052","article-title":"ToMBench: Benchmarking theory of mind in large language models","author":"Chen","year":"2024","journal-title":"arXiv"},{"key":"2026090412050628100_bib17","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3359164","article-title":"Efficient elicitation approaches to estimate collective crowd answers","volume":"3(CSCW)","author":"Chung","year":"2019","journal-title":"Proceedings of the ACM on Human-Computer Interaction"},{"key":"2026090412050628100_bib18","doi-asserted-by":"publisher","first-page":"869","DOI":"10.1145\/3600211.3604692","article-title":"Human uncertainty in concept-based AI systems","volume-title":"Proceedings of the 2023 AAAI\/ACM Conference on AI, Ethics, and Society","author":"Collins","year":"2023"},{"key":"2026090412050628100_bib19","first-page":"454","article-title":"Human-in-the-loop mixup","volume-title":"Proceedings of the Thirty-Ninth Conference on Uncertainty in Artificial Intelligence","author":"Collins","year":"2023"},{"key":"2026090412050628100_bib20","doi-asserted-by":"publisher","first-page":"293","DOI":"10.1609\/aies.v7i1.31637","article-title":"Beyond thumbs up\/down: Untangling challenges of fine-grained feedback for text-to-image generation","author":"Collins","year":"2024"},{"issue":"10","key":"2026090412050628100_bib21","doi-asserted-by":"publisher","first-page":"1851","DOI":"10.1038\/s41562-024-01991-9","article-title":"Building machines that learn and think with people","volume":"8","author":"Collins","year":"2024","journal-title":"Nature Human Behaviour"},{"issue":"4","key":"2026090412050628100_bib22","doi-asserted-by":"publisher","first-page":"281","DOI":"10.1037\/h0040957","article-title":"Construct validity in psychological tests","volume":"52","author":"Cronbach","year":"1955","journal-title":"Psychological Bulletin"},{"issue":"11","key":"2026090412050628100_bib23","doi-asserted-by":"publisher","first-page":"688","DOI":"10.1038\/s44159-023-00241-5","article-title":"Using large language models in psychology","volume":"2","author":"Demszky","year":"2023","journal-title":"Nature Reviews Psychology"},{"key":"2026090412050628100_bib24","doi-asserted-by":"publisher","first-page":"301","DOI":"10.1109\/HRI.2013.6483603","article-title":"Legibility and predictability of robot motion","volume-title":"Proceedings of the 8th ACM\/IEEE International Conference on Human-Robot Interaction (HRI 2013),","author":"Dragan","year":"2013"},{"issue":"1","key":"2026090412050628100_bib25","doi-asserted-by":"publisher","first-page":"2319","DOI":"10.1038\/s41467-019-10301-1","article-title":"Uncovering the structure of self-regulation through data-driven ontology discovery","volume":"10","author":"Eisenberg","year":"2019","journal-title":"Nature Communications"},{"issue":"43","key":"2026090412050628100_bib26","doi-asserted-by":"publisher","first-page":"26562","DOI":"10.1073\/pnas.1905334117","article-title":"Performance vs. competence in human-machine comparisons","volume":"117","author":"Firestone","year":"2020","journal-title":"Proceedings of the National Academy of Sciences"},{"key":"2026090412050628100_bib27","doi-asserted-by":"publisher","first-page":"144","DOI":"10.1007\/978-3-030-43089-4_10","article-title":"Generating plans that predict themselves","volume-title":"Algorithmic foundations of robotics XII","author":"Fisac","year":"2020"},{"issue":"8","key":"2026090412050628100_bib28","doi-asserted-by":"publisher","first-page":"451","DOI":"10.1038\/s44159-023-00211-x","article-title":"Baby steps in evaluating the capacities of large language models","volume":"2","author":"Frank","year":"2023","journal-title":"Nature Reviews Psychology"},{"issue":"1480","key":"2026090412050628100_bib29","doi-asserted-by":"publisher","first-page":"659","DOI":"10.1098\/rstb.2006.2002","article-title":"Before and below \u2018theory of mind\u2019: Embodied simulation and the neural correlates of social cognition","volume":"362","author":"Gallese","year":"2007","journal-title":"Philosophical Transactions of the Royal Society B: Biological Sciences"},{"key":"2026090412050628100_bib30","article-title":"Understanding social reasoning in language models with language models","volume":"36","author":"Gandhi","year":"2024","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2026090412050628100_bib31","doi-asserted-by":"publisher","first-page":"6577","DOI":"10.18653\/v1\/2024.naacl-long.366","article-title":"A survey of confidence estimation and calibration in large language models","volume-title":"Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL 2024)","author":"Geng","year":"2024"},{"issue":"1","key":"2026090412050628100_bib32","doi-asserted-by":"publisher","first-page":"13","DOI":"10.1007\/s10515-023-00409-6","article-title":"Can ai serve as a substitute for human subjects in software engineering research?","volume":"31","author":"Gerosa","year":"2024","journal-title":"Automated Software Engineering"},{"key":"2026090412050628100_bib33","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3491102.3502004","article-title":"Jury learning: Integrating dissenting voices into machine learning models","volume-title":"Proceedings of the 2022 CHI Conference on Human Factors in Computing Systems (CHI 2022)","author":"Gordon","year":"2022"},{"issue":"5","key":"2026090412050628100_bib34","doi-asserted-by":"publisher","first-page":"1029","DOI":"10.1037\/a0015141","article-title":"Liberals and conservatives rely on different sets of moral foundations","volume":"96","author":"Graham","year":"2009","journal-title":"Journal of Personality and Social Psychology"},{"key":"2026090412050628100_bib35","doi-asserted-by":"publisher","first-page":"125","DOI":"10.1016\/j.copsyc.2015.09.007","article-title":"Cultural differences in moral judgment and behavior, across and within societies","volume":"8","author":"Graham","year":"2016","journal-title":"Current Opinion in Psychology"},{"issue":"11","key":"2026090412050628100_bib36","doi-asserted-by":"publisher","first-page":"873","DOI":"10.1016\/j.tics.2020.09.001","article-title":"Understanding human intelligence through human limitations","volume":"24","author":"Griffiths","year":"2020","journal-title":"Trends in Cognitive Sciences"},{"key":"2026090412050628100_bib37","doi-asserted-by":"crossref","DOI":"10.21428\/e2759450.7b420317","volume-title":"Bayesian models of cognition: Reverse engineering the mind","author":"Griffiths","year":"2024"},{"key":"2026090412050628100_bib38","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3544548.3580688","article-title":"Evaluating large language models in generating synthetic HCI research data: A case study","volume-title":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems.","author":"H\u00e4m\u00e4l\u00e4inen","year":"2023"},{"issue":"5","key":"2026090412050628100_bib39","doi-asserted-by":"publisher","first-page":"733","DOI":"10.1521\/soco.2009.27.5.733","article-title":"Adaptive rationality: An evolutionary perspective on cognitive bias","volume":"27","author":"Haselton","year":"2009","journal-title":"Social Cognition"},{"issue":"2\u20133","key":"2026090412050628100_bib40","doi-asserted-by":"publisher","first-page":"61","DOI":"10.1017\/S0140525X0999152X","article-title":"The weirdest people in the world?","volume":"33","author":"Henrich","year":"2010","journal-title":"Behavioral and Brain Sciences"},{"key":"2026090412050628100_bib41","doi-asserted-by":"publisher","first-page":"397","DOI":"10.1007\/s10462-016-9505-7","article-title":"Evaluation in artificial intelligence: From task-oriented to ability-oriented measurement","volume":"48","author":"Hern\u00e1ndez-Orallo","year":"2017","journal-title":"Artificial Intelligence Review"},{"key":"2026090412050628100_bib42","doi-asserted-by":"publisher","first-page":"33","DOI":"10.1146\/annurev-control-042920-015547","article-title":"Cognitive science as a source of forward and inverse models of human decisions for robotics and control","volume":"5","author":"Ho","year":"2022","journal-title":"Annual Review of Control, Robotics, and Autonomous Systems"},{"key":"2026090412050628100_bib43","article-title":"Auxiliary task demands mask the capabilities of smaller language models","volume-title":"Proceedings of the First Conference on Language Modeling (COLM 2024)","author":"Hu","year":"2024"},{"key":"2026090412050628100_bib44","doi-asserted-by":"publisher","first-page":"5040","DOI":"10.18653\/v1\/2023.emnlp-main.306","article-title":"Prompting is not a substitute for probability measurements in large language models","volume-title":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing (EMNLP 2023)","author":"Hu","year":"2023"},{"issue":"1932","key":"2026090412050628100_bib45","doi-asserted-by":"publisher","first-page":"20230499","DOI":"10.1098\/rstb.2023.0499","article-title":"Re-evaluating theory of mind evaluation in large language models","volume":"380","author":"Hu","year":"2025","journal-title":"Philosophical Transactions of the Royal Society B: Biological Sciences"},{"issue":"2","key":"2026090412050628100_bib46","doi-asserted-by":"publisher","first-page":"230","DOI":"10.1038\/s41562-024-02096-z","article-title":"How to evaluate the cognitive abilities of LLMs","volume":"9","author":"Ivanova","year":"2025","journal-title":"Nature Human Behaviour"},{"key":"2026090412050628100_bib47","doi-asserted-by":"publisher","first-page":"5075","DOI":"10.18653\/v1\/2023.emnlp-main.308","article-title":"Stop uploading test data in plain text: Practical strategies for mitigating data contamination by evaluation benchmarks","volume-title":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing (EMNLP 2023)","author":"Jacovi","year":"2023"},{"key":"2026090412050628100_bib48","doi-asserted-by":"publisher","first-page":"101334","DOI":"10.1016\/j.cogpsych.2020.101334","article-title":"The na\u00efve utility calculus as a unified, quantitative framework for action understanding","volume":"123","author":"Jara-Ettinger","year":"2020","journal-title":"Cognitive Psychology"},{"key":"2026090412050628100_bib49","first-page":"16077","article-title":"MMToM-QA: Multimodal theory of mind question answering","volume-title":"Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics","author":"Jin","year":"2024"},{"issue":"7873","key":"2026090412050628100_bib50","doi-asserted-by":"publisher","first-page":"583","DOI":"10.1038\/s41586-021-03819-2","article-title":"Highly accurate protein structure prediction with AlphaFold","volume":"596","author":"Jumper","year":"2021","journal-title":"Nature"},{"issue":"3","key":"2026090412050628100_bib51","doi-asserted-by":"publisher","first-page":"217","DOI":"10.1016\/0001-6918(91)90036-Y","article-title":"Calibration and probability judgements: Conceptual and methodological issues","volume":"77","author":"Keren","year":"1991","journal-title":"Acta Psychologica"},{"key":"2026090412050628100_bib52","doi-asserted-by":"crossref","DOI":"10.52202\/079017-3342","article-title":"The PRISM alignment dataset: What participatory, representative and individualised human feedback reveals about the subjective and multicultural alignment of large language models","volume-title":"Advances in Neural Information Processing Systems 37 (NeurIPS 2024)","author":"Kirk","year":"2024"},{"issue":"45","key":"2026090412050628100_bib53","doi-asserted-by":"publisher","first-page":"e2405460121","DOI":"10.1073\/pnas.2405460121","article-title":"Evaluating large language models in theory of mind tasks","volume":"121","author":"Kosinski","year":"2024","journal-title":"Proceedings of the National Academy of Sciences"},{"issue":"13","key":"2026090412050628100_bib54","doi-asserted-by":"publisher","first-page":"R545","DOI":"10.1016\/j.cub.2015.04.053","article-title":"Striking individual differences in color perception uncovered by \u2018the dress\u2019 photograph","volume":"25","author":"Lafer-Sousa","year":"2015","journal-title":"Current Biology"},{"key":"2026090412050628100_bib55","doi-asserted-by":"publisher","first-page":"e253","DOI":"10.1017\/S0140525X16001837","article-title":"Building machines that learn and think like people","volume":"40","author":"Lake","year":"2017","journal-title":"Behavioral and Brain Sciences"},{"issue":"6677","key":"2026090412050628100_bib56","doi-asserted-by":"publisher","first-page":"1416","DOI":"10.1126\/science.adi2336","article-title":"Learning skillful medium-range global weather forecasting","volume":"382","author":"Lam","year":"2023","journal-title":"Science"},{"issue":"4","key":"2026090412050628100_bib57","doi-asserted-by":"publisher","first-page":"1441","DOI":"10.1162\/coli_a_00525","article-title":"Can language models handle recursively nested grammatical structures? A case study on comparing models and humans","volume":"50","author":"Lampinen","year":"2024","journal-title":"Computational Linguistics"},{"issue":"7","key":"2026090412050628100_bib58","doi-asserted-by":"publisher","first-page":"pgae233","DOI":"10.1093\/pnasnexus\/pgae233","article-title":"Language models, like humans, show content effects on reasoning tasks","volume":"3","author":"Lampinen","year":"2024","journal-title":"Proceedings of the National Academy of Sciences"},{"key":"2026090412050628100_bib59","doi-asserted-by":"publisher","first-page":"5872","DOI":"10.18653\/v1\/D19-1598","article-title":"Revisiting the evaluation of theory of mind through question answering","volume-title":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)","author":"Le","year":"2019"},{"issue":"42","key":"2026090412050628100_bib60","doi-asserted-by":"publisher","first-page":"26158","DOI":"10.1073\/pnas.2014505117","article-title":"The logic of universalization guides moral judgment","volume":"117","author":"Levine","year":"2020","journal-title":"Proceedings of the National Academy of Sciences"},{"key":"2026090412050628100_bib61","doi-asserted-by":"publisher","first-page":"e1","DOI":"10.1017\/S0140525X1900061X","article-title":"Resource-rational analysis: Understanding human cognition as the optimal use of limited computational resources","volume":"43","author":"Lieder","year":"2019","journal-title":"Behavioral and Brain Sciences"},{"key":"2026090412050628100_bib62","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2406.17055","article-title":"Large language models assume people are more rational than we really are","author":"Liu","year":"2024","journal-title":"arXiv"},{"key":"2026090412050628100_bib63","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2311.00687","author":"Liu","year":"2023","journal-title":"arXiv"},{"key":"2026090412050628100_bib64","volume-title":"Vision: A computational investigation into the human representation and processing of visual information","author":"Marr","year":"1982"},{"issue":"9","key":"2026090412050628100_bib65","doi-asserted-by":"publisher","first-page":"741","DOI":"10.1037\/0003-066X.50.9.741","article-title":"Validity of psychological assessment: Validation of inferences from persons\u2019 responses and performances as scientific inquiry into score meaning","volume":"50","author":"Messick","year":"1995","journal-title":"American Psychologist"},{"key":"2026090412050628100_bib66","volume-title":"Society of mind","author":"Minsky","year":"1988"},{"key":"2026090412050628100_bib67","first-page":"6710","volume-title":"The turing test and our shifting conceptions of intelligence","author":"Mitchell","year":"2024"},{"key":"2026090412050628100_bib68","article-title":"Human alignment of neural network representations","volume-title":"The Eleventh International Conference on Learning Representations (ICLR 2023)","author":"Muttenthaler","year":"2022"},{"key":"2026090412050628100_bib69","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2409.06509","article-title":"Aligning machine and human visual representations across abstraction levels","author":"Muttenthaler","year":"2024","journal-title":"arXiv"},{"key":"2026090412050628100_bib70","first-page":"50978","article-title":"Improving neural network representations using human similarity judgments","volume-title":"Advances in Neural Information Processing Systems","author":"Muttenthaler","year":"2023"},{"key":"2026090412050628100_bib71","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2103.14749","article-title":"Pervasive label errors in test sets destabilize machine learning benchmarks","author":"Northcutt","year":"2021","journal-title":"arXiv"},{"issue":"Supp. 1","key":"2026090412050628100_bib72","doi-asserted-by":"publisher","first-page":"69","DOI":"10.1080\/00031305.2018.1518265","article-title":"Expert knowledge elicitation: Subjective but scientific","volume":"73","author":"O\u2019Hagan","year":"2019","journal-title":"American Statistician"},{"key":"2026090412050628100_bib73","doi-asserted-by":"publisher","DOI":"10.1002\/0470033312","volume-title":"Uncertain judgements: Eliciting expert probabilities","author":"O\u2019Hagan","year":"2006"},{"key":"2026090412050628100_bib74","doi-asserted-by":"publisher","first-page":"1291","DOI":"10.1162\/opmi_a_00168","article-title":"When new experience leads to new knowledge: A computational framework for formalizing epistemically transformative experiences","volume":"8","author":"Ongchoco","year":"2024","journal-title":"Open Mind: Discoveries in Cognitive Science"},{"key":"2026090412050628100_bib75","article-title":"Generative agent simulations of 1,000 people","author":"Park","year":"2024","journal-title":"arXiv"},{"key":"2026090412050628100_bib76","first-page":"9617","article-title":"Human uncertainty makes classification more robust","volume-title":"Proceedings of the IEEE\/CVF International Conference on Computer Vision (ICCV 2019)","author":"Peterson","year":"2019"},{"key":"2026090412050628100_bib77","article-title":"Principles of animal cognition for LLM evaluations: A case study on transitive inference","volume-title":"NeurIPS 2024 Workshop on Behavioral Machine Learning","author":"Rane","year":"2024"},{"issue":"6","key":"2026090412050628100_bib78","doi-asserted-by":"publisher","first-page":"386","DOI":"10.1037\/h0042519","article-title":"The perceptron: A probabilistic model for information storage and organization in the brain","volume":"65","author":"Rosenblatt","year":"1958","journal-title":"Psychological Review"},{"key":"2026090412050628100_bib79","doi-asserted-by":"publisher","first-page":"318","DOI":"10.7551\/mitpress\/4943.003.0128","article-title":"\u201cLearning internal representations by error propagation,\u201d parallel distributed processing: Explorations in the microstructures of cognition","volume-title":"Neurocomputing, volume 1: Foundations of research","author":"Rumelhart","year":"1988"},{"key":"2026090412050628100_bib80","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2207.06214","article-title":"Is one annotation enough? A data-centric image classification benchmark for noisy and ambiguous label estimation","author":"Schmarje","year":"2022","journal-title":"arXiv"},{"key":"2026090412050628100_bib81","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3613904.3642159","article-title":"Rehearsal: Simulating conflict to teach conflict resolution","volume-title":"Proceedings of the 2024 CHI Conference on Human Factors in Computing Systems.Citation Identifiers","author":"Shaikh","year":"2024"},{"issue":"10","key":"2026090412050628100_bib82","doi-asserted-by":"publisher","first-page":"e2300963120","DOI":"10.1073\/pnas.2300963120","article-title":"Probing the psychology of AI models","volume":"120","author":"Shiffrin","year":"2023","journal-title":"Proceedings of the National Academy of Sciences"},{"key":"2026090412050628100_bib83","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2402.05070","article-title":"A roadmap to pluralistic alignment","author":"Sorensen","year":"2024","journal-title":"arXiv"},{"key":"2026090412050628100_bib84","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2206.04615","article-title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","author":"Srivastava","year":"2022","journal-title":"arXiv"},{"issue":"11","key":"2026090412050628100_bib85","doi-asserted-by":"publisher","first-page":"e2111547119","DOI":"10.1073\/pnas.2111547119","article-title":"Bayesian modeling of human-AI complementarity","volume":"119","author":"Steyvers","year":"2022","journal-title":"Proceedings of the National Academy of Sciences"},{"key":"2026090412050628100_bib86","first-page":"2036","article-title":"On the informativeness of supervision signals","volume-title":"Proceedings of the Thirty-Ninth Conference on Uncertainty in Artificial Intelligence","author":"Sucholutsky","year":"2023"},{"key":"2026090412050628100_bib87","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2310.13018","article-title":"Getting aligned on representational alignment","author":"Sucholutsky","year":"2023","journal-title":"arXiv"},{"issue":"9","key":"2026090412050628100_bib88","doi-asserted-by":"publisher","first-page":"738","DOI":"10.1016\/j.tics.2022.06.002","article-title":"The signature-testing approach to mapping biological and artificial intelligences","volume":"26","author":"Taylor","year":"2022","journal-title":"Trends in Cognitive Sciences"},{"key":"2026090412050628100_bib89","article-title":"Bayesian modeling of human concept learning","volume":"11","author":"Tenenbaum","year":"1998","journal-title":"Advances in Neural Information Processing Systems"},{"issue":"236","key":"2026090412050628100_bib90","doi-asserted-by":"publisher","first-page":"433","DOI":"10.1093\/mind\/LIX.236.433","article-title":"Computing machinery and intelligence","volume":"LIX","author":"Turing","year":"1950","journal-title":"Mind"},{"issue":"4157","key":"2026090412050628100_bib91","doi-asserted-by":"publisher","first-page":"1124","DOI":"10.1126\/science.185.4157.1124","article-title":"Judgment under uncertainty: Heuristics and biases: Biases in judgments reveal some heuristics of thinking under uncertainty","volume":"185","author":"Tversky","year":"1974","journal-title":"Science"},{"key":"2026090412050628100_bib92","doi-asserted-by":"publisher","first-page":"818451","DOI":"10.3389\/frai.2022.818451","article-title":"Scaling and disagreements: Bias, noise, and ambiguity","volume":"5","author":"Uma","year":"2022","journal-title":"Frontiers in Artificial Intelligence"},{"issue":"1","key":"2026090412050628100_bib93","doi-asserted-by":"publisher","first-page":"173","DOI":"10.1609\/hcomp.v8i1.7478","article-title":"A case for soft loss functions","volume":"8","author":"Uma","year":"2020","journal-title":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing"},{"key":"2026090412050628100_bib94","article-title":"Bringing comparative cognition to computers","author":"Voudouris","year":"2024","journal-title":"Trends in Cognitive Sciences"},{"issue":"1","key":"2026090412050628100_bib95","doi-asserted-by":"publisher","first-page":"36","DOI":"10.1145\/365153.365168","article-title":"ELIZA\u2014A computer program for the study of natural language communication between man and machine","volume":"9","author":"Weizenbaum","year":"1966","journal-title":"Communications of the ACM"},{"issue":"1","key":"2026090412050628100_bib96","doi-asserted-by":"publisher","first-page":"103","DOI":"10.1016\/0010-0277(83)90004-5","article-title":"Beliefs about beliefs: Representation and constraining function of wrong beliefs in young children\u2019s understanding of deception","volume":"13","author":"Wimmer","year":"1983","journal-title":"Cognition"},{"issue":"4","key":"2026090412050628100_bib97","doi-asserted-by":"publisher","first-page":"343","DOI":"10.1037\/1076-898X.2.4.343","article-title":"Measuring psychological uncertainty: Verbal versus numeric methods","volume":"2","author":"Windschitl","year":"1996","journal-title":"Journal of Experimental Psychology: Applied"},{"key":"2026090412050628100_bib98","doi-asserted-by":"crossref","first-page":"59008","DOI":"10.52202\/075280-2574","article-title":"Fine-grained human feedback gives better rewards for language model training","volume":"36","author":"Wu","year":"2023","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2026090412050628100_bib99","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2409.10849","article-title":"Pragmatic embodied spoken instruction following in human-robot collaboration with theory of mind","author":"Ying","year":"2024","journal-title":"arXiv"},{"key":"2026090412050628100_bib100","first-page":"26","article-title":"Inferring the goals of communicating agents from actions and instructions","volume-title":"Proceedings of the AAAI Symposium Series","author":"Ying","year":"2023"},{"key":"2026090412050628100_bib101","first-page":"2094","article-title":"Pragmatic instruction following and goal assistance via cooperative language-guided inverse planning","volume-title":"Proceedings of the 23rd International Conference on Autonomous Agents and Multiagent Systems","author":"Zhi-Xuan","year":"2024"},{"issue":"8032","key":"2026090412050628100_bib102","doi-asserted-by":"publisher","first-page":"61","DOI":"10.1038\/s41586-024-07930-y","article-title":"Larger and more instructable language models become less reliable","volume":"634","author":"Zhou","year":"2024","journal-title":"Nature"}],"container-title":["Open Mind"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/direct.mit.edu\/opmi\/article-pdf\/doi\/10.1162\/OPMI.a.385\/2625713\/opmi.a.385.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/direct.mit.edu\/opmi\/article-pdf\/doi\/10.1162\/OPMI.a.385\/2625713\/opmi.a.385.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,9,4]],"date-time":"2026-09-04T16:05:25Z","timestamp":1788537925000},"score":1,"resource":{"primary":{"URL":"https:\/\/direct.mit.edu\/opmi\/article\/doi\/10.1162\/OPMI.a.385\/138849\/On-Benchmarking-Human-like-Intelligence-in"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"references-count":102,"URL":"https:\/\/doi.org\/10.1162\/opmi.a.385","relation":{},"ISSN":["2470-2986"],"issn-type":[{"value":"2470-2986","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2026]]},"published":{"date-parts":[[2026]]}}}