{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,13]],"date-time":"2026-02-13T16:10:32Z","timestamp":1770999032394,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":42,"publisher":"ACM","funder":[{"name":"Institute of Information & communications Technology Planning & Evaluation (IITP) grant","award":["NO.RS-2021-II211343"],"award-info":[{"award-number":["NO.RS-2021-II211343"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,2,18]]},"DOI":"10.1145\/3770762.3772536","type":"proceedings-article","created":{"date-parts":[[2026,2,13]],"date-time":"2026-02-13T15:15:58Z","timestamp":1770995758000},"page":"45-51","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Deriving Instructional Insights from Human-LLM Co-Evaluation of Student Collaboration in Data-Centric Programming"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0005-5165-640X","authenticated-orcid":false,"given":"Marshall","family":"An","sequence":"first","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-2825-2280","authenticated-orcid":false,"given":"Christine","family":"Kwon","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3762-0639","authenticated-orcid":false,"given":"Yoonjae","family":"Lee","sequence":"additional","affiliation":[{"name":"Seoul National University, Seoul, Republic of Korea"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4065-1476","authenticated-orcid":false,"given":"Ji-Hyeon","family":"Hur","sequence":"additional","affiliation":[{"name":"Seoul National University, Seoul, Republic of Korea"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-5941-9208","authenticated-orcid":false,"given":"Dongho","family":"Lee","sequence":"additional","affiliation":[{"name":"Seoul National University, Seoul, Republic of Korea"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-7909-5421","authenticated-orcid":false,"given":"Vincent","family":"Huai","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-1538-8186","authenticated-orcid":false,"given":"Barry","family":"Zheng","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-6064-2418","authenticated-orcid":false,"given":"Matthew","family":"Yu","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-6077-2998","authenticated-orcid":false,"given":"Joana","family":"Liu","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-6259-8502","authenticated-orcid":false,"given":"Jenny","family":"Pugh","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3268-477X","authenticated-orcid":false,"given":"Gahgene","family":"Gweon","sequence":"additional","affiliation":[{"name":"Seoul National University, Seoul, Republic of Korea"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2291-1468","authenticated-orcid":false,"given":"John","family":"Stamper","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2026,2,17]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1007\/s40593-013-0012-6"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"crossref","unstructured":"Toufique Ahmed Premkumar Devanbu Christoph Treude and Michael Pradel. 2025. Can LLMs Replace Manual Annotation of Software Engineering Artifacts? arXiv:2408.05534 [cs.SE] https:\/\/arxiv.org\/abs\/2408.05534","DOI":"10.1109\/MSR66628.2025.00086"},{"key":"e_1_3_2_1_3_1","volume-title":"How to Reduce Bias in the Life Cycle of a Data Science Project. Journal of Business and Psychology","author":"Banks George C","year":"2025","unstructured":"George C Banks, Scott Tonidandel,Wenwen Dou, Matthew J Gerson, Depeng Xu, and Jill E Yavorsky. 2025. How to Reduce Bias in the Life Cycle of a Data Science Project. Journal of Business and Psychology (2025), 1--21."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1111\/jcal.12208"},{"key":"e_1_3_2_1_5_1","volume-title":"Qualitative inquiry and research design: Choosing among five approaches","author":"Creswell John W","unstructured":"John W Creswell and Cheryl N Poth. 2016. Qualitative inquiry and research design: Choosing among five approaches. Sage publications."},{"key":"e_1_3_2_1_6_1","volume-title":"Over-scripting CSCL: The risks of blending collaborative learning with instructional design. Three worlds of CSCL. Can we support CSCL?","author":"Dillenbourg Pierre","year":"2002","unstructured":"Pierre Dillenbourg. 2002. Over-scripting CSCL: The risks of blending collaborative learning with instructional design. Three worlds of CSCL. Can we support CSCL? (2002), 61--91."},{"key":"e_1_3_2_1_7_1","volume-title":"Learning in humans and machine: Towards an interdisciplinary learning science","author":"Dillenbourg P.","unstructured":"P. Dillenbourg, M. Baker, A. Blaye, and C. O'Malley. 1996. The evolution of research on collaborative learning. In Learning in humans and machine: Towards an interdisciplinary learning science, E. Spada and P. Reiman (Eds.). Elsevier, Oxford, 189--211."},{"key":"e_1_3_2_1_8_1","first-page":"111","article-title":"Computer-supported collaborative learning: The basics","volume":"21","author":"Dillenbourg Pierre","year":"2007","unstructured":"Pierre Dillenbourg and Frank Fischer. 2007. Computer-supported collaborative learning: The basics. Zeitschrift f\u00fcr Berufs-und Wirtschaftsp\u00e4dagogik 21 (2007), 111--130.","journal-title":"Zeitschrift f\u00fcr Berufs-und Wirtschaftsp\u00e4dagogik"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11412-007-9033-1"},{"key":"e_1_3_2_1_10_1","unstructured":"P Dillenbourg and P Jermann. 2007. SWISH: A model for designing CSCL scripts. Scripting computer-supported collaborative learning--Cognitive computational and educational perspectives. Computer-supported collaborative learning series. New York: Springer (2007)."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1365-2729.2007.00191.x"},{"key":"e_1_3_2_1_12_1","unstructured":"Martina M. Echtenbruck Simone F\u00fchles-Ubach Boris Naujoks and Elisabeth Kaliva. 2025. A Data Literacy Competence Model for Higher Education and Research. arXiv:2504.15690 [cs.CY] https:\/\/arxiv.org\/abs\/2504.15690"},{"key":"e_1_3_2_1_13_1","unstructured":"Ananya Ganesh Chelsea Chandler Sidney D'Mello Martha Palmer and Katharina von der Wense. 2024. Prompting as Panacea? A Case Study of In-Context Learning Performance for Qualitative Coding of Classroom Dialog. (2024)."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3636555.3636879"},{"key":"e_1_3_2_1_15_1","volume-title":"Nan Duan, and Weizhu Chen.","author":"He Xingwei","year":"2024","unstructured":"Xingwei He, Zhenghao Lin, Yeyun Gong, A-Long Jin, Hang Zhang, Chen Lin, Jian Jiao, Siu Ming Yiu, Nan Duan, and Weizhu Chen. 2024. AnnoLLM: Making Large Language Models to Be Better Crowdsourced Annotators. arXiv:2303.16854 [cs.CL] https:\/\/arxiv.org\/abs\/2303.16854"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"Markus Hisch and Xing David Wang. 2024. Using LLMs to label medical papers according to the CIViC evidence model. arXiv:2407.04466 [cs.CL] https:\/\/arxiv.org\/abs\/2407.04466","DOI":"10.31219\/osf.io\/brxnp"},{"key":"e_1_3_2_1_17_1","volume-title":"International Conference on Immersive Learning. Springer, 287--301","author":"Holly Michael","year":"2024","unstructured":"Michael Holly, Jannik Hildebrandt, and Johanna Pirker. 2024. A Computer-Supported Collaborative Learning Environment for Computer Science Education. In International Conference on Immersive Learning. Springer, 287--301."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3636555.3636910"},{"key":"e_1_3_2_1_19_1","volume-title":"Ten years of computer-supported collaborative learning: A meta-analysis of CSCL in STEM education during 2005--2014. Educational research review 28","author":"Jeong Heisawn","year":"2019","unstructured":"Heisawn Jeong, Cindy E Hmelo-Silver, and Kihyun Jo. 2019. Ten years of computer-supported collaborative learning: A meta-analysis of CSCL in STEM education during 2005--2014. Educational research review 28 (2019), 100284."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-94-017-0781-7_8"},{"key":"e_1_3_2_1_21_1","volume-title":"Agile data science 2.0: Building full-stack data analytics applications with spark. '' O'Reilly Media","author":"Jurney Russell","unstructured":"Russell Jurney. 2017. Agile data science 2.0: Building full-stack data analytics applications with spark. '' O'Reilly Media, Inc.''."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1002\/acp.1730"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11412-018-9277-y"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.naacl-long.228"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.22318\/cscl2025.611229"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/174666.174668"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.bdr.2020.100183"},{"key":"e_1_3_2_1_28_1","volume-title":"PISA 2015 collaborative problem solving framework. 131--188 pages.","author":"OECD.","year":"2017","unstructured":"OECD. 2017. PISA 2015 collaborative problem solving framework. 131--188 pages."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-85098-1_5"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3636555.3636922"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1108\/ILS-04-2020-0095"},{"key":"e_1_3_2_1_32_1","volume-title":"Information-processing theory of human problem solving","author":"Simon Herbert A","unstructured":"Herbert A Simon. 1978. Information-processing theory of human problem solving. Erlbaum Hillsdale, NJ."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.compedu.2003.10.004"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.compedu.2019.103672"},{"key":"e_1_3_2_1_35_1","volume-title":"Psychology of learning and motivation.","author":"Sweller John","unstructured":"John Sweller. 2011. Cognitive load theory. In Psychology of learning and motivation. Vol. 55. Elsevier, 37--76."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"crossref","unstructured":"Zhen Tan Dawei Li Song Wang Alimohammad Beigi Bohan Jiang Amrita Bhattacharjee Mansooreh Karami Jundong Li Lu Cheng and Huan Liu. 2024. Large Language Models for Data Annotation and Synthesis: A Survey. arXiv:2402.13446 [cs.CL] https:\/\/arxiv.org\/abs\/2402.13446","DOI":"10.18653\/v1\/2024.emnlp-main.54"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"crossref","unstructured":"Shuohang Wang Yang Liu Yichong Xu Chenguang Zhu and Michael Zeng. 2021. Want To Reduce Labeling Cost? GPT-3 Can Help. arXiv:2108.13487 [cs.CL] https:\/\/arxiv.org\/abs\/2108.13487","DOI":"10.18653\/v1\/2021.findings-emnlp.354"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3641960"},{"key":"e_1_3_2_1_39_1","volume-title":"Denny Zhou, et al.","author":"Wei Jason","year":"2022","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Fei Xia, Ed Chi, Quoc V Le, Denny Zhou, et al. 2022. Chain-of-thought prompting elicits reasoning in large language models. Advances in neural information processing systems 35 (2022), 24824--24837."},{"key":"e_1_3_2_1_40_1","volume-title":"Proceedings of the 4th international conference on the practical applications of knowledge discovery and data mining","volume":"1","author":"Wirth R\u00fcdiger","year":"2000","unstructured":"R\u00fcdiger Wirth and Jochen Hipp. 2000. CRISP-DM: Towards a standard process model for data mining. In Proceedings of the 4th international conference on the practical applications of knowledge discovery and data mining, Vol. 1. Manchester, 29--39."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"crossref","unstructured":"Xiaoyu Zhang Yishan Li Jiayin Wang Bowen Sun Weizhi Ma Peijie Sun and Min Zhang. 2024. Large Language Models as Evaluators for Recommendation Explanations. arXiv:2406.03248 [cs.IR] https:\/\/arxiv.org\/abs\/2406.03248","DOI":"10.1145\/3640457.3688075"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"crossref","unstructured":"Mingqian Zheng Jiaxin Pei Lajanugen Logeswaran Moontae Lee and David Jurgens. 2024. When ''A Helpful Assistant'' Is Not Really Helpful: Personas in System Prompts Do Not Improve Performances of Large Language Models. arXiv:2311.10054 [cs.CL] https:\/\/arxiv.org\/abs\/2311.10054","DOI":"10.18653\/v1\/2024.findings-emnlp.888"}],"event":{"name":"SIGCSE TS 2026:The 57th ACM Technical Symposium on Computer Science Education","location":"St. Louis MO USA","sponsor":["SIGCSE ACM Special Interest Group on Computer Science Education"]},"container-title":["Proceedings of the 57th ACM Technical Symposium on Computer Science Education V.1"],"original-title":[],"deposited":{"date-parts":[[2026,2,13]],"date-time":"2026-02-13T15:23:49Z","timestamp":1770996229000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3770762.3772536"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,2,17]]},"references-count":42,"alternative-id":["10.1145\/3770762.3772536","10.1145\/3770762"],"URL":"https:\/\/doi.org\/10.1145\/3770762.3772536","relation":{},"subject":[],"published":{"date-parts":[[2026,2,17]]},"assertion":[{"value":"2026-02-17","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}