{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,10]],"date-time":"2026-07-10T16:37:11Z","timestamp":1783701431276,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":47,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,3]],"date-time":"2024-07-03T00:00:00Z","timestamp":1719964800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100006374","name":"University of Toronto","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,3]]},"DOI":"10.1145\/3649217.3653554","type":"proceedings-article","created":{"date-parts":[[2024,7,3]],"date-time":"2024-07-03T18:30:20Z","timestamp":1720031420000},"page":"388-393","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":29,"title":["Can Small Language Models With Retrieval-Augmented Generation Replace Large Language Models When Learning Computer Science?"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0004-5558-256X","authenticated-orcid":false,"given":"Suqing","family":"Liu","sequence":"first","affiliation":[{"name":"University of Toronto Mississauga, Mississauga, Ontario, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-8472-3517","authenticated-orcid":false,"given":"Zezhu","family":"Yu","sequence":"additional","affiliation":[{"name":"University of Toronto Mississauga, Mississauga, Ontario, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-9793-2355","authenticated-orcid":false,"given":"Feiran","family":"Huang","sequence":"additional","affiliation":[{"name":"University of Toronto Mississauga, Mississauga, Ontario, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-9843-1643","authenticated-orcid":false,"given":"Yousef","family":"Bulbulia","sequence":"additional","affiliation":[{"name":"University of Toronto Mississauga, Mississauga, Ontario, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6557-2534","authenticated-orcid":false,"given":"Andreas","family":"Bergen","sequence":"additional","affiliation":[{"name":"University of Toronto Mississauga, Mississauga, Ontario, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2965-5302","authenticated-orcid":false,"given":"Michael","family":"Liut","sequence":"additional","affiliation":[{"name":"University of Toronto Mississauga, Mississauga, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,7,3]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proceedings of the 18th Workshop on Innovative Use of NLP for Building Educational Applications (BEA","author":"Al-Hossami Erfan","year":"2023","unstructured":"Erfan Al-Hossami, Razvan Bunescu, Ryan Teehan, Laurel Powell, and Khyati Mahajan et al. 2023. Socratic questioning of novice debuggers: A benchmark dataset and preliminary evaluations. In Proceedings of the 18th Workshop on Innovative Use of NLP for Building Educational Applications (BEA 2023). 709--726."},{"key":"e_1_3_2_1_2_1","volume-title":"Sebastian Borgeaud, Yonghui Wu, Jean-Baptiste Alayrac, Jiahui Yu, Radu Soricut, Johan Schalkwyk, Andrew M. Dai, Anja Hauth, Katie Millican, et al.","author":"Google Gemini Team","year":"2023","unstructured":"Gemini Team Google: Rohan Anil, Sebastian Borgeaud, Yonghui Wu, Jean-Baptiste Alayrac, Jiahui Yu, Radu Soricut, Johan Schalkwyk, Andrew M. Dai, Anja Hauth, Katie Millican, et al. 2023. Gemini: a family of highly capable multimodal models. arxiv: 2312.11805 [cs.CL]"},{"key":"e_1_3_2_1_3_1","volume-title":"Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection. arXiv preprint arXiv:2310.11511","author":"Asai Akari","year":"2023","unstructured":"Akari Asai, Zeqiu Wu, Yizhong Wang, Avirup Sil, and Hannaneh Hajishirzi. 2023. Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection. arXiv preprint arXiv:2310.11511 (2023)."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3582515.3609555"},{"key":"e_1_3_2_1_5_1","volume-title":"Post training 4-bit quantization of convolutional networks for rapid-deployment. Advances in Neural Information Processing Systems","author":"Banner Ron","year":"2019","unstructured":"Ron Banner, Yury Nahshan, Elad Hoffer, and Daniel Soudry. 2019. Post training 4-bit quantization of convolutional networks for rapid-deployment. Advances in Neural Information Processing Systems (2019)."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.5220\/0012007500003470"},{"key":"e_1_3_2_1_7_1","volume-title":"Evaluating Large Language Models for Document-grounded Response Generation in Information-Seeking Dialogues. arXiv preprint arXiv:2309.11838","author":"Braunschweiler Norbert","year":"2023","unstructured":"Norbert Braunschweiler, Rama Doddipatla, Simon Keizer, and Svetlana Stoyanchev. 2023. Evaluating Large Language Models for Document-grounded Response Generation in Information-Seeking Dialogues. arXiv preprint arXiv:2309.11838 (2023)."},{"key":"e_1_3_2_1_8_1","unstructured":"Tom B. Brown Benjamin Mann Nick Ryder Melanie Subbiah and Jared Kaplan et al. 2020. Language models are few-shot learners. Advances in neural information processing systems Vol. 33 (2020)."},{"key":"e_1_3_2_1_9_1","volume-title":"Benchmarking large language models in retrieval-augmented generation. arXiv preprint arXiv:2309.01431","author":"Chen Jiawei","year":"2023","unstructured":"Jiawei Chen, Hongyu Lin, Xianpei Han, and Le Sun. 2023. Benchmarking large language models in retrieval-augmented generation. arXiv preprint arXiv:2309.01431 (2023)."},{"key":"e_1_3_2_1_10_1","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan and Henrique Ponde de Oliveira Pinto et al. 2021. Evaluating large language models trained on code. arXiv preprint arXiv:2107.03374 (2021)."},{"key":"e_1_3_2_1_11_1","volume-title":"Cohen","author":"Chen Wenhu","year":"2022","unstructured":"Wenhu Chen, Hexiang Hu, Chitwan Saharia, and William W. Cohen. 2022. Re-imagen: Retrieval-augmented text-to-image generator. arXiv preprint arXiv:2209.14491 (2022)."},{"key":"e_1_3_2_1_12_1","unstructured":"Hyung Won Chung Le Hou Shayne Longpre Barret Zoph and Yi Tay et al. 2022. Scaling instruction-finetuned language models. arXiv preprint arXiv:2210.11416 (2022)."},{"key":"e_1_3_2_1_13_1","volume-title":"Desmarais et al","author":"Dakhel Arghavan Moradi","year":"2023","unstructured":"Arghavan Moradi Dakhel, Vahid Majdinasab, Amin Nikanjam, Foutse Khomh, and Michel C. Desmarais et al. 2023. Github copilot ai pair programmer: Asset or liability? Journal of Systems and Software (2023)."},{"key":"e_1_3_2_1_14_1","volume-title":"Flashattention-2: Faster attention with better parallelism and work partitioning. arXiv preprint arXiv:2307.08691","author":"Dao Tri","year":"2023","unstructured":"Tri Dao. 2023. Flashattention-2: Faster attention with better parallelism and work partitioning. arXiv preprint arXiv:2307.08691 (2023)."},{"key":"e_1_3_2_1_15_1","volume-title":"Md Faisal Mahbub Chowdhury, and Alfio Gliozzo","author":"Glass Michael","year":"2021","unstructured":"Michael Glass, Gaetano Rossiello, Md Faisal Mahbub Chowdhury, and Alfio Gliozzo. 2021. Robust retrieval augmented generation for zero-shot slot filling. arXiv preprint arXiv:2108.13934 (2021)."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"Arto Hellas Juho Leinonen Sami Sarsa Charles Koutcheme and Lilja Kujanp\u00e4\u00e4 et al. 2023. Exploring the Responses of Large Language Models to Beginner Programmers' Help Requests. arXiv preprint arXiv:2306.05715 (2023).","DOI":"10.1145\/3568813.3600139"},{"key":"e_1_3_2_1_17_1","unstructured":"Yann Hicke Anmol Agarwal Qianou Ma and Paul Denny. 2023. AI-TA: Towards an Intelligent Question-Answer Teaching Assistant using Open-Source LLMs. arxiv: 2311.02775 [cs.LG]"},{"key":"e_1_3_2_1_18_1","unstructured":"Albert Q. Jiang Alexandre Sablayrolles Arthur Mensch Chris Bamford and Devendra Singh Chaplot et al. 2023 a. Mistral 7B. arXiv preprint arXiv:2310.06825."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Zhengbao Jiang Frank F. Xu Luyu Gao Zhiqing Sun and Qian Liu et al. 2023 b. Active retrieval augmented generation. arXiv preprint arXiv:2305.06983.","DOI":"10.18653\/v1\/2023.emnlp-main.495"},{"key":"e_1_3_2_1_20_1","unstructured":"Jared Kaplan Sam McCandlish Tom Henighan Tom B. Brown and Benjamin Chess et al. 2020. Scaling laws for neural language models. arXiv preprint arXiv:2001.08361 (2020)."},{"key":"e_1_3_2_1_21_1","volume-title":"Iulian Vlad Serban, and Ekaterina Kochmar","author":"Kulshreshtha Devang","year":"2022","unstructured":"Devang Kulshreshtha, Muhammad Shayan, Robert Belfer, Siva Reddy, Iulian Vlad Serban, and Ekaterina Kochmar. 2022. Few-shot Question Generation for Personalized Feedback in Intelligent Tutoring Systems. arXiv preprint arXiv:2206.04187 (2022)."},{"key":"e_1_3_2_1_22_1","volume-title":"2023 a. Impact of Guidance and Interaction Strategies for LLM Use on Learner Performance and Perception. arXiv preprint arXiv:2310.13712","author":"Kumar Harsh","year":"2023","unstructured":"Harsh Kumar, Ilya Musabirov, Mohi Reza, Jiakai Shi, and Anastasia Kuzminykh et al. 2023 a. Impact of Guidance and Interaction Strategies for LLM Use on Learner Performance and Perception. arXiv preprint arXiv:2310.13712 (2023)."},{"key":"e_1_3_2_1_23_1","volume-title":"Anastasia Kuzminykh, and Michael Liut.","author":"Kumar Harsh","year":"2024","unstructured":"Harsh Kumar, Ilya Musabirov, Mohi Reza, Jiakai Shi, Xinyuan Wang, Joseph Jay Williams, Anastasia Kuzminykh, and Michael Liut. 2024. Impact of Guidance and Interaction Strategies for LLM Use on Learner Performance and Perception. arxiv: 2310.13712 [cs.HC]"},{"key":"e_1_3_2_1_24_1","volume-title":"Workshop on Partnerships for Co-Creating Educational Content at the Learning Analytics and Knowledge Conference","author":"Kumar Harsh","year":"2023","unstructured":"Harsh Kumar, Ilya Musabirov, Joseph Jay Williams, and Michael Liut. 2023 b. QuickTA: Exploring the Design Space of Using Large Language Models to Provide Support to Students. In Workshop on Partnerships for Co-Creating Educational Content at the Learning Analytics and Knowledge Conference 2023. Arlington, TX, USA."},{"key":"e_1_3_2_1_25_1","volume-title":"Albert: A lite bert for self-supervised learning of language representations. arxiv","author":"Lan Zhenzhong","year":"2019","unstructured":"Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, and Piyush Sharma et al. 2019. Albert: A lite bert for self-supervised learning of language representations. arxiv: 1909.11942 [cs.CL]"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"crossref","unstructured":"Juho Leinonen Paul Denny Stephen MacNeil Sami Sarsa and Seth Bernstein et al. 2023 a. Comparing code explanations created by students and large language models. arxiv: 2304.03938 [cs.CY]","DOI":"10.1145\/3587102.3588785"},{"key":"e_1_3_2_1_27_1","volume-title":"Proceedings of the 54th ACM Technical Symposium on Computer Science Education V. 1. 563--569","author":"Leinonen Juho","unstructured":"Juho Leinonen, Arto Hellas, Sami Sarsa, Brent Reeves, and Paul Denny et al. 2023 b. Using large language models to enhance programming error messages. In Proceedings of the 54th ACM Technical Symposium on Computer Science Education V. 1. 563--569."},{"key":"e_1_3_2_1_28_1","first-page":"9459","article-title":"Retrieval-augmented generation for knowledge-intensive nlp tasks","volume":"33","author":"Lewis Patrick","year":"2020","unstructured":"Patrick Lewis, Ethan Perez, Aleksandra Piktus, Fabio Petroni, and Vladimir Karpukhin et al. 2020. Retrieval-augmented generation for knowledge-intensive nlp tasks. Advances in Neural Information Processing Systems , Vol. 33 (2020), 9459--9474.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_29_1","volume-title":"Eliciting human preferences with language models. arXiv preprint arXiv:2310.11589","author":"Li Belinda Z.","year":"2023","unstructured":"Belinda Z. Li, Alex Tamkin, Noah Goodman, and Jacob Andreas. 2023. Eliciting human preferences with language models. arXiv preprint arXiv:2310.11589 (2023)."},{"key":"e_1_3_2_1_30_1","volume-title":"Codehelp: Using large language models with guardrails for scalable support in programming classes. arxiv: 2308.06921 [cs.CY]","author":"Liffiton Mark","year":"2023","unstructured":"Mark Liffiton, Brad Sheese, Jaromir Savelka, and Paul Denny. 2023. Codehelp: Using large language models with guardrails for scalable support in programming classes. arxiv: 2308.06921 [cs.CY]"},{"key":"e_1_3_2_1_31_1","unstructured":"Ziyang Luo Can Xu Pu Zhao Xiubo Geng and Chongyang Tao et al. 2023. Augmented Large Language Models with Parametric Knowledge Guiding. arXiv preprint arXiv:2305.04757 (2023)."},{"key":"e_1_3_2_1_32_1","volume-title":"Codegen: An open large language model for code with multi-turn program synthesis. arXiv preprint arXiv:2203.13474","author":"Nijkamp Erik","year":"2022","unstructured":"Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, and Huan Wang et al. 2022. Codegen: An open large language model for code with multi-turn program synthesis. arXiv preprint arXiv:2203.13474 (2022)."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF00168958"},{"key":"e_1_3_2_1_34_1","volume-title":"Baker","author":"Pankiewicz Maciej","year":"2023","unstructured":"Maciej Pankiewicz and Ryan S. Baker. 2023. Large Language Models (GPT) for automating feedback on programming assignments. arXiv preprint arXiv:2307.00150 (2023)."},{"key":"e_1_3_2_1_35_1","volume-title":"Saikat Chakraborty, Baishakhi Ray, and Kai-Wei Chang.","author":"Parvez Md Rizwan","year":"2021","unstructured":"Md Rizwan Parvez, Wasi Uddin Ahmad, Saikat Chakraborty, Baishakhi Ray, and Kai-Wei Chang. 2021. Retrieval augmented code generation and summarization. arXiv preprint arXiv:2108.11601 (2021)."},{"key":"e_1_3_2_1_36_1","volume-title":"Proceedings of the 2023 Working Group Reports on Innovation and Technology in Computer Science Education. 108--159","author":"Prather James","unstructured":"James Prather, Paul Denny, Juho Leinonen, Brett A. Becker, and Ibrahim Albluwi et al. 2023. The robots are here: Navigating the generative ai revolution in computing education. In Proceedings of the 2023 Working Group Reports on Innovation and Technology in Computer Science Education. 108--159."},{"key":"e_1_3_2_1_37_1","unstructured":"Jack W. Rae Sebastian Borgeaud Trevor Cai Katie Millican and Jordan Hoffmann et al. 2021. Scaling language models: Methods analysis & insights from training gopher. arXiv preprint arXiv:2112.11446 (2021)."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"crossref","unstructured":"Timo Schick and Hinrich Sch\u00fctze. 2020a. Exploiting cloze questions for few shot text classification and natural language inference. arXiv preprint arXiv:2001.07676.","DOI":"10.18653\/v1\/2021.eacl-main.20"},{"key":"e_1_3_2_1_39_1","volume-title":"It's not just size that matters: Small language models are also few-shot learners. arXiv preprint arXiv:2009.07118","author":"Schick Timo","year":"2020","unstructured":"Timo Schick and Hinrich Sch\u00fctze. 2020b. It's not just size that matters: Small language models are also few-shot learners. arXiv preprint arXiv:2009.07118 (2020)."},{"key":"e_1_3_2_1_40_1","volume-title":"Proceedings of the 2nd international conference on learning analytics and knowledge. 252--254","author":"Siemens George","unstructured":"George Siemens and Ryan S. J. d. Baker. 2012. Learning analytics and educational data mining: towards communication and collaboration. In Proceedings of the 2nd international conference on learning analytics and knowledge. 252--254."},{"key":"e_1_3_2_1_41_1","unstructured":"Hugo Touvron Louis Martin Kevin Stone Peter Albert and Amjad Almahairi et al. 2023. Llama 2: Open foundation and fine-tuned chat models. arXiv preprint arXiv:2307.09288 (2023)."},{"key":"e_1_3_2_1_42_1","volume-title":"Chi conference on human factors in computing systems extended abstracts. 1--7.","author":"Vaithilingam Priyan","unstructured":"Priyan Vaithilingam, Tianyi Zhang, and Elena L. Glassman. 2022. Expectation vs. experience: Evaluating the usability of code generation tools powered by large language models. In Chi conference on human factors in computing systems extended abstracts. 1--7."},{"key":"e_1_3_2_1_43_1","volume-title":"Proceedings of the 2021 International Conference on Management of Data","author":"Wang Jianguo","unstructured":"Jianguo Wang, Xiaomeng Yi, Rentong Guo, Hai Jin, and et al. 2021. Milvus: A Purpose-Built Vector Data Management System. In Proceedings of the 2021 International Conference on Management of Data (Virtual Event, China) (SIGMOD `21). 2614--2627."},{"key":"e_1_3_2_1_44_1","unstructured":"Jason Wei Yi Tay Rishi Bommasani Colin Raffel and Barret Zoph et al. 2022. Emergent abilities of large language models. arXiv preprint arXiv:2206.07682. Available online at https:\/\/arxiv.org\/abs\/2206.07682."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1109\/CIE.2002.1185885"},{"key":"e_1_3_2_1_46_1","volume-title":"An introduction to artificial intelligence in education","author":"Yu Shengquan","unstructured":"Shengquan Yu and Yu Lu. 2021. An introduction to artificial intelligence in education. Springer."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3409073.3409079"}],"event":{"name":"ITiCSE 2024: Innovation and Technology in Computer Science Education","location":"Milan Italy","acronym":"ITiCSE 2024","sponsor":["SIGCSE ACM Special Interest Group on Computer Science Education"]},"container-title":["Proceedings of the 2024 on Innovation and Technology in Computer Science Education V. 1"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3649217.3653554","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3649217.3653554","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,21]],"date-time":"2025-08-21T14:50:39Z","timestamp":1755787839000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3649217.3653554"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,3]]},"references-count":47,"alternative-id":["10.1145\/3649217.3653554","10.1145\/3649217"],"URL":"https:\/\/doi.org\/10.1145\/3649217.3653554","relation":{},"subject":[],"published":{"date-parts":[[2024,7,3]]},"assertion":[{"value":"2024-07-03","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}