{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,23]],"date-time":"2026-04-23T08:01:18Z","timestamp":1776931278026,"version":"3.51.2"},"publisher-location":"New York, NY, USA","reference-count":68,"publisher":"ACM","license":[{"start":{"date-parts":[[2027,4,13]],"date-time":"2027-04-13T00:00:00Z","timestamp":1807574400000},"content-version":"vor","delay-in-days":365,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["CNS-2213791, 2414915"],"award-info":[{"award-number":["CNS-2213791, 2414915"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,4,13]]},"DOI":"10.1145\/3772318.3791283","type":"proceedings-article","created":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T05:14:30Z","timestamp":1776057270000},"page":"1-22","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Not Everyone Wins with LLMs: Behavioral Patterns and Pedagogical Implications for AI Literacy in Programmatic Data Science"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-8634-130X","authenticated-orcid":false,"given":"Qianou","family":"Ma","sequence":"first","affiliation":[{"name":"Human-Computer-Interaction Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5850-4768","authenticated-orcid":false,"given":"Kenneth R","family":"Koedinger","sequence":"additional","affiliation":[{"name":"Human Computer Interaction Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1630-0588","authenticated-orcid":false,"given":"Tongshuang","family":"Wu","sequence":"additional","affiliation":[{"name":"Human-Computer Interaction Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2026,4,13]]},"reference":[{"key":"e_1_3_3_2_2_2","doi-asserted-by":"crossref","unstructured":"Ravinithesh Annapureddy Alessandro Fornaroli and Daniel Gatica-Perez. 2025. Generative AI literacy: Twelve defining competencies. Digital Government: Research and Practice 6 1 (2025) 1\u201321.","DOI":"10.1145\/3685680"},{"key":"e_1_3_3_2_3_2","doi-asserted-by":"publisher","unstructured":"Douglas Bates Martin M\u00e4chler Ben Bolker and Steve Walker. 2015. Fitting linear mixed-effects models Usinglme4. J. Stat. Softw. 67 1 (Oct. 2015) 1\u201348. 10.18637\/jss.v067.i01","DOI":"10.18637\/jss.v067.i01"},{"key":"e_1_3_3_2_4_2","unstructured":"Joel Becker Nate Rush Elizabeth Barnes and David Rein. 2025. Measuring the impact of early-2025 AI on experienced open-source developer productivity. arXiv [cs.AI] (July 2025). arxiv:https:\/\/arXiv.org\/abs\/2507.09089\u00a0[cs.AI] http:\/\/arxiv.org\/abs\/2507.09089"},{"key":"e_1_3_3_2_5_2","doi-asserted-by":"crossref","unstructured":"Virginia Braun and Victoria Clarke. 2006. Using thematic analysis in psychology. Qualitative research in psychology 3 2 (2006) 77\u2013101.","DOI":"10.1191\/1478088706qp063oa"},{"key":"e_1_3_3_2_6_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642377"},{"key":"e_1_3_3_2_7_2","unstructured":"Jiaqi Chen Yanzhe Zhang Yutong Zhang Yijia Shao and Diyi Yang. 2025. Generative Interfaces for Language Models. arXiv [cs.CL] (Aug. 2025). arxiv:https:\/\/arXiv.org\/abs\/2508.19227\u00a0[cs.CL] http:\/\/arxiv.org\/abs\/2508.19227"},{"key":"e_1_3_3_2_8_2","doi-asserted-by":"publisher","DOI":"10.1145\/3706598.3714002"},{"key":"e_1_3_3_2_9_2","unstructured":"Zixin Chen Jiachen Wang Meng Xia Kento Shigyo Dingdong Liu Rong Zhang and Huamin Qu. 2024. StuGPTViz: A visual analytics approach to understand student-ChatGPT interactions. arXiv [cs.HC] (July 2024). arxiv:https:\/\/arXiv.org\/abs\/2407.12423\u00a0[cs.HC] http:\/\/arxiv.org\/abs\/2407.12423"},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"crossref","unstructured":"Ruijia Cheng Titus Barik Alan Leung Fred Hohman and Jeffrey Nichols. 2024. BISCUIT: Scaffolding LLM-generated code with ephemeral UIs in computational notebooks. arXiv [cs.HC] (April 2024). arxiv:https:\/\/arXiv.org\/abs\/2404.07387\u00a0[cs.HC] http:\/\/arxiv.org\/abs\/2404.07387","DOI":"10.1109\/VL\/HCC60511.2024.00012"},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"publisher","unstructured":"Michelene T\u00a0H Chi and Ruth Wylie. 2014. The ICAP Framework: Linking Cognitive Engagement to Active Learning Outcomes. Educ. Psychol. 49 4 (Oct. 2014) 219\u2013243. 10.1080\/00461520.2014.965823","DOI":"10.1080\/00461520.2014.965823"},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"crossref","unstructured":"Mark Claypool David Brown Phong Le and Makoto Waseda. 2001. Inferring user interest. IEEE Internet Computing 5 6 (2001) 32\u201339.","DOI":"10.1109\/4236.968829"},{"key":"e_1_3_3_2_13_2","volume-title":"Mental models in human-computer interaction: Research issues about what the user of software knows","author":"Council National\u00a0Research","year":"1987","unstructured":"National\u00a0Research Council and Others. 1987. Mental models in human-computer interaction: Research issues about what the user of software knows. nap.nationalacademies.org. https:\/\/nap.nationalacademies.org\/catalog\/790\/mental-models-in-human-computer-interaction-research-issues-about-what"},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"publisher","unstructured":"Louis Deslauriers Logan\u00a0S McCarty Kelly Miller Kristina Callaghan and Greg Kestin. 2019. Measuring actual learning versus feeling of learning in response to being actively engaged in the classroom. Proc. Natl. Acad. Sci. U. S. A. 116 39 (Sept. 2019) 19251\u201319257. 10.1073\/pnas.1821936116","DOI":"10.1073\/pnas.1821936116"},{"key":"e_1_3_3_2_15_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4939-0378-8_14"},{"key":"e_1_3_3_2_16_2","doi-asserted-by":"crossref","unstructured":"Jennifer Fereday and Eimear Muir-Cochrane. 2006. Demonstrating rigor using thematic analysis: A hybrid approach of inductive and deductive coding and theme development. International journal of qualitative methods 5 1 (2006) 80\u201392.","DOI":"10.1177\/160940690600500107"},{"key":"e_1_3_3_2_17_2","unstructured":"Google. 2025. Google Colab. https:\/\/colab.research.google.com\/. Accessed: 2025-9-8."},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","DOI":"10.1145\/3641554.3701859"},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"publisher","unstructured":"Michael Greenacre Patrick J\u00a0F Groenen Trevor Hastie Alfonso\u00a0Iodice D\u2019Enza Angelos Markos and Elena Tuzhilina. 2022. Principal component analysis. Nat. Rev. Methods Primers 2 1 (Dec. 2022) 100. 10.1038\/s43586-022-00184-w","DOI":"10.1038\/s43586-022-00184-w"},{"key":"e_1_3_3_2_20_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3641891"},{"key":"e_1_3_3_2_21_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.naacl-long.255"},{"key":"e_1_3_3_2_22_2","doi-asserted-by":"publisher","DOI":"10.1016\/s0166-4115(08)62386-9"},{"key":"e_1_3_3_2_23_2","doi-asserted-by":"publisher","DOI":"10.1145\/964442.964480"},{"key":"e_1_3_3_2_24_2","doi-asserted-by":"publisher","DOI":"10.1145\/2348283.2348313"},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","DOI":"10.1145\/2858796.2858798"},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580919"},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/3654777.3676345"},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","unstructured":"Majeed Kazemitabaar Runlong Ye Xiaoning Wang Austin\u00a0Z Henley Paul Denny Michelle Craig and Tovi Grossman. 2024. CodeAid: Evaluating a Classroom Deployment of an LLM-based Programming Assistant that Balances Student and Educator Needs. arXiv [cs.HC] (Jan. 2024). 10.1145\/3613904.3642773 arxiv:https:\/\/arXiv.org\/abs\/2401.11314\u00a0[cs.HC]","DOI":"10.1145\/3613904.3642773"},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"publisher","DOI":"10.1145\/3706599.3707611"},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"crossref","unstructured":"Michal Kosinski. 2024. Evaluating large language models in theory of mind tasks. Proceedings of the National Academy of Sciences 121 45 (2024) e2405460121.","DOI":"10.1073\/pnas.2405460121"},{"key":"e_1_3_3_2_31_2","doi-asserted-by":"publisher","unstructured":"David\u00a0R Krathwohl. 2002. A revision of bloom\u2019s taxonomy: An overview. Theory Pract. 41 4 (Nov. 2002) 212\u2013218. 10.1207\/s15430421tip4104_2","DOI":"10.1207\/s15430421tip4104_2"},{"key":"e_1_3_3_2_32_2","doi-asserted-by":"publisher","unstructured":"Justin Kruger and David Dunning. 1999. Unskilled and unaware of it: How difficulties in recognizing one\u2019s own incompetence lead to inflated self-assessments. J. Pers. Soc. Psychol. 77 6 (1999) 1121\u20131134. 10.1037\/0022-3514.77.6.1121","DOI":"10.1037\/0022-3514.77.6.1121"},{"key":"e_1_3_3_2_33_2","doi-asserted-by":"publisher","DOI":"10.1145\/3657604.3662042"},{"key":"e_1_3_3_2_34_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3502030"},{"key":"e_1_3_3_2_35_2","doi-asserted-by":"publisher","unstructured":"Ivan Letteri and Pierpaolo Vittorini. 2025. Enhancing student feedback in data science education: Harnessing the power of AI-generated approaches. Int. J. Artif. Intell. Educ. (June 2025) 1\u201324. 10.1007\/s40593-025-00492-8","DOI":"10.1007\/s40593-025-00492-8"},{"key":"e_1_3_3_2_36_2","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3608128"},{"key":"e_1_3_3_2_37_2","unstructured":"Nelson Lojo Rafael Gonz\u00e1lez Rohan Philip Jos\u00e9\u00a0Antonio Parejo Amador\u00a0Dur\u00e1n Toro Armando Fox and Pablo Fern\u00e1ndez. 2025. Using large language models to develop requirements Elicitation skills. arXiv [cs.SE] (March 2025). arxiv:https:\/\/arXiv.org\/abs\/2503.07800\u00a0[cs.SE] http:\/\/arxiv.org\/abs\/2503.07800"},{"key":"e_1_3_3_2_38_2","doi-asserted-by":"publisher","unstructured":"Christopher Lum Guoxuan Xu and Sam Lau. 2025. \u201cI\u2019m not sure but... \u201d: Expert Practices that Enable Effective Code Comprehension in Data Science Vol.\u00a01. Association for Computing Machinery. 10.1145\/3641554.3701933","DOI":"10.1145\/3641554.3701933"},{"key":"e_1_3_3_2_39_2","doi-asserted-by":"publisher","unstructured":"Qianou Ma Weirui Peng Chenyang Yang Hua Shen Ken Koedinger and Tongshuang Wu. 2025. What should we engineer in prompts? Training humans in requirement-driven LLM use. ACM Trans. Comput. Hum. Interact. 32 4 (Aug. 2025) 1\u201327. 10.1145\/3731756","DOI":"10.1145\/3731756"},{"key":"e_1_3_3_2_40_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-64302-6_19"},{"key":"e_1_3_3_2_41_2","doi-asserted-by":"publisher","unstructured":"Katerina\u00a0M Marcoulides and Tenko Raykov. 2019. Evaluation of variance inflation factors in regression models using latent variable modeling methods. Educ. Psychol. Meas. 79 5 (Oct. 2019) 874\u2013882. 10.1177\/0013164418817803","DOI":"10.1177\/0013164418817803"},{"key":"e_1_3_3_2_42_2","doi-asserted-by":"publisher","unstructured":"Mary\u00a0L McHugh. 2012. Interrater reliability: the kappa statistic. Biochem. Med. (Zagreb) 22 3 (2012) 276\u2013282. 10.11613\/bm.2012.031","DOI":"10.11613\/bm.2012.031"},{"key":"e_1_3_3_2_43_2","doi-asserted-by":"publisher","DOI":"10.1002\/9780470479216.corpsy0524"},{"key":"e_1_3_3_2_44_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580940"},{"key":"e_1_3_3_2_45_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642706"},{"key":"e_1_3_3_2_46_2","unstructured":"Shramay Palta Nirupama Chandrasekaran Rachel Rudinger and Scott Counts. 2025. Speaking the right language: The impact of expertise alignment in user-ai interactions. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2502.18685 (2025)."},{"key":"e_1_3_3_2_47_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP66354.2025.00060"},{"key":"e_1_3_3_2_48_2","unstructured":"Sida Peng Eirini Kalliamvakou Peter Cihon and Mert Demirer. 2023. The impact of ai on developer productivity: Evidence from github copilot. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2302.06590 (2023)."},{"key":"e_1_3_3_2_49_2","doi-asserted-by":"publisher","DOI":"10.1145\/3706598.3713271"},{"key":"e_1_3_3_2_50_2","unstructured":"Brett Puppart and Jaan Aru. 2025. Short-term AI literacy intervention does not reduce over-reliance on incorrect ChatGPT recommendations. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2503.10556 (2025)."},{"key":"e_1_3_3_2_51_2","first-page":"4218","volume-title":"International conference on machine learning","author":"Rabinowitz Neil","year":"2018","unstructured":"Neil Rabinowitz, Frank Perbet, Francis Song, Chiyuan Zhang, SM\u00a0Ali Eslami, and Matthew Botvinick. 2018. Machine theory of mind. In International conference on machine learning. PMLR, 4218\u20134227."},{"key":"e_1_3_3_2_52_2","doi-asserted-by":"publisher","DOI":"10.1145\/3641554.3701863"},{"key":"e_1_3_3_2_53_2","doi-asserted-by":"publisher","unstructured":"Ghadeer Sawalha Imran Taj and Abdulhadi Shoufan. 2024. Analyzing student prompts and their effect on ChatGPT\u2019s performance. Cogent Educ. 11 1 (Dec. 2024). 10.1080\/2331186x.2024.2397200","DOI":"10.1080\/2331186x.2024.2397200"},{"key":"e_1_3_3_2_54_2","doi-asserted-by":"publisher","DOI":"10.1109\/fie61694.2024.10893442"},{"key":"e_1_3_3_2_55_2","doi-asserted-by":"crossref","unstructured":"Fangchen Song Ashish Agarwal and Wen Wen. 2024. The impact of generative AI on collaborative open-source software development: Evidence from GitHub Copilot. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2410.02091 (2024).","DOI":"10.2139\/ssrn.4856935"},{"key":"e_1_3_3_2_56_2","doi-asserted-by":"publisher","DOI":"10.1145\/3689535.3689553"},{"key":"e_1_3_3_2_57_2","doi-asserted-by":"crossref","unstructured":"James\u00a0WA Strachan Dalila Albergo Giulia Borghini Oriana Pansardi Eugenio Scaliti Saurabh Gupta Krati Saxena Alessandro Rufo Stefano Panzeri Guido Manzi et\u00a0al. 2024. Testing theory of mind in large language models and humans. Nature Human Behaviour (2024) 1\u201311.","DOI":"10.21203\/rs.3.rs-3262385\/v1"},{"key":"e_1_3_3_2_58_2","unstructured":"Anselm Strauss and Juliet Corbin. 1998. Basics of qualitative research techniques. (1998)."},{"key":"e_1_3_3_2_59_2","doi-asserted-by":"publisher","unstructured":"Sebastian Tassoti. 2024. Assessment of students use of generative artificial intelligence: Prompting strategies and prompt engineering in chemistry education. J. Chem. Educ. 101 6 (June 2024) 2475\u20132482. 10.1021\/acs.jchemed.4c00212","DOI":"10.1021\/acs.jchemed.4c00212"},{"key":"e_1_3_3_2_60_2","doi-asserted-by":"crossref","unstructured":"Edward Tenner. 2015. The design of everyday things by Donald Norman. Technology and Culture 56 3 (2015) 785\u2013787.","DOI":"10.1353\/tech.2015.0104"},{"key":"e_1_3_3_2_61_2","unstructured":"Jiessie Tie Bingsheng Yao Tianshi Li Syed\u00a0Ishtiaque Ahmed Dakuo Wang and Shurui Zhou. 2024. LLMs are imperfect then what? An empirical study on LLM failures in software engineering. arXiv [cs.SE] (Nov. 2024). arxiv:https:\/\/arXiv.org\/abs\/2411.09916\u00a0[cs.SE] http:\/\/arxiv.org\/abs\/2411.09916"},{"key":"e_1_3_3_2_62_2","doi-asserted-by":"publisher","unstructured":"Endel Tulving and Zena Pearlstone. 1966. Availability versus accessibility of information in memory for words. J. Verbal Learning Verbal Behav. 5 4 (Aug. 1966) 381\u2013391. 10.1016\/s0022-5371(66)80048-8","DOI":"10.1016\/s0022-5371(66)80048-8"},{"key":"e_1_3_3_2_63_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-98465-5_9"},{"key":"e_1_3_3_2_64_2","doi-asserted-by":"crossref","unstructured":"Vijay Viswanathan Kiril Gashteovski Kiril Gashteovski Carolin Lawrence Tongshuang Wu and Graham Neubig. 2024. Large language models enable few-shot clustering. Transactions of the Association for Computational Linguistics 12 (2024) 321\u2013333.","DOI":"10.1162\/tacl_a_00648"},{"key":"e_1_3_3_2_65_2","doi-asserted-by":"publisher","unstructured":"April\u00a0Yi Wang Dakuo Wang Jaimie Drozdal Michael Muller Soya Park Justin\u00a0D Weisz Xuye Liu Lingfei Wu and Casey Dugan. 2022. Documentation matters: Human-centered AI system to assist data science code documentation in computational notebooks. ACM Trans. Comput. Hum. Interact. 29 2 (April 2022) 1\u201333. 10.1145\/3489465","DOI":"10.1145\/3489465"},{"key":"e_1_3_3_2_66_2","volume-title":"Understanding by Design","author":"Wiggins Grant\u00a0P","year":"2005","unstructured":"Grant\u00a0P Wiggins and Jay McTighe. 2005. Understanding by Design. ASCD, USA. https:\/\/play.google.com\/store\/books\/details?id=N2EfKlyUN4QC"},{"key":"e_1_3_3_2_67_2","unstructured":"Jinjin Zhao Avidgor Gal and Sanjay Krishnan. 2024. Data makes better data scientists. arXiv [cs.HC] (May 2024). arxiv:https:\/\/arXiv.org\/abs\/2405.17690\u00a0[cs.HC] http:\/\/arxiv.org\/abs\/2405.17690"},{"key":"e_1_3_3_2_68_2","doi-asserted-by":"publisher","DOI":"10.1145\/3585059.3611431"},{"key":"e_1_3_3_2_69_2","doi-asserted-by":"publisher","DOI":"10.1145\/3696630.3731663"}],"event":{"name":"CHI 2026: CHI Conference on Human Factors in Computing Systems","location":"Barcelona Spain","acronym":"CHI '26","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3772318.3791283","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3772318.3791283","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,17]],"date-time":"2026-04-17T10:17:46Z","timestamp":1776421066000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3772318.3791283"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4,13]]},"references-count":68,"alternative-id":["10.1145\/3772318.3791283","10.1145\/3772318"],"URL":"https:\/\/doi.org\/10.1145\/3772318.3791283","relation":{},"subject":[],"published":{"date-parts":[[2026,4,13]]},"assertion":[{"value":"2026-04-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}