{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,21]],"date-time":"2026-03-21T15:56:23Z","timestamp":1774108583346,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":27,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,12,9]],"date-time":"2023-12-09T00:00:00Z","timestamp":1702080000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,12,9]]},"DOI":"10.1145\/3627217.3627235","type":"proceedings-article","created":{"date-parts":[[2023,12,19]],"date-time":"2023-12-19T19:37:52Z","timestamp":1703014672000},"page":"104-107","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["Evaluating Copilot on CS1 Code Writing Problems with Suppressed Specifications"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0006-4690-8307","authenticated-orcid":false,"given":"Varshini","family":"Venkatesh","sequence":"first","affiliation":[{"name":"SSN University, India"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-1169-8686","authenticated-orcid":false,"given":"Vaishnavi","family":"Venkatesh","sequence":"additional","affiliation":[{"name":"SRM University, India"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2252-0141","authenticated-orcid":false,"given":"Viraj","family":"Kumar","sequence":"additional","affiliation":[{"name":"Indian Institute of Science, India"}]}],"member":"320","published-online":{"date-parts":[[2023,12,19]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Grounded Copilot: How Programmers Interact with Code-Generating Models. arxiv:2206.15000\u00a0[cs.HC]","author":"Barke Shraddha","year":"2022","unstructured":"Shraddha Barke, Michael\u00a0B. James, and Nadia Polikarpova. 2022. Grounded Copilot: How Programmers Interact with Code-Generating Models. arxiv:2206.15000\u00a0[cs.HC]"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569759"},{"key":"e_1_3_2_1_3_1","volume-title":"2021. Evaluating Large Language Models Trained on Code. CoRR abs\/2107.03374","author":"Chen Mark","year":"2021","unstructured":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique\u00a0Pond\u00e9 de Oliveira\u00a0Pinto, Jared Kaplan, Harrison Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, and Mikhail\u00a0Pavlov et al.2021. Evaluating Large Language Models Trained on Code. CoRR abs\/2107.03374 (2021). arXiv:2107.03374https:\/\/arxiv.org\/abs\/2107.03374"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1028174.971375"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3552490.3552494"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/2828959.2828967"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569823"},{"key":"e_1_3_2_1_8_1","volume-title":"The Robots Are Coming: Exploring the Implications of OpenAI Codex on Introductory Programming. In Australasian Computing Education Conference. 10\u201319","author":"Finnie-Ansley James","year":"2022","unstructured":"James Finnie-Ansley, Paul Denny, Brett\u00a0A Becker, Andrew Luxton-Reilly, and James Prather. 2022. The Robots Are Coming: Exploring the Implications of OpenAI Codex on Introductory Programming. In Australasian Computing Education Conference. 10\u201319."},{"key":"e_1_3_2_1_9_1","volume-title":"Retrieved","author":"Horstmann S.","year":"2023","unstructured":"Cay\u00a0S. Horstmann. [n. d.]. CodeCheck Python Exercises. Retrieved May 12, 2023 from https:\/\/horstmann.com\/codecheck\/python-questions.html"},{"key":"e_1_3_2_1_10_1","volume-title":"Computer Science Curricula 2013: Curriculum Guidelines for Undergraduate Degree Programs in Computer Science","author":"Association for Computing Machinery\u00a0(ACM) Joint Task Force\u00a0on Computing\u00a0Curricula and IEEE\u00a0Computer Society. 2013.","unstructured":"Association for Computing Machinery\u00a0(ACM) Joint Task Force\u00a0on Computing\u00a0Curricula and IEEE\u00a0Computer Society. 2013. Computer Science Curricula 2013: Curriculum Guidelines for Undergraduate Degree Programs in Computer Science. Association for Computing Machinery, New York, NY, USA."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545947.3569591"},{"key":"e_1_3_2_1_12_1","unstructured":"Shuvendu\u00a0K. Lahiri Aaditya Naik Georgios Sakkas Piali Choudhury Curtis von Veh Madanlal Musuvathi Jeevana\u00a0Priya Inala Chenglong Wang and Jianfeng Gao. 2022. Interactive Code Generation via Test-Driven User-Intent Formalization. arxiv:2208.05950\u00a0[cs.SE]"},{"key":"e_1_3_2_1_13_1","volume-title":"Competition-level code generation with alphacode. arXiv preprint arXiv:2203.07814","author":"Li Yujia","year":"2022","unstructured":"Yujia Li, David Choi, Junyoung Chung, Nate Kushman, Julian Schrittwieser, R\u00e9mi Leblond, Tom Eccles, James Keeling, Felix Gimeno, Agustin\u00a0Dal Lago, 2022. Competition-level code generation with alphacode. arXiv preprint arXiv:2203.07814 (2022)."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2960310.2960334"},{"key":"e_1_3_2_1_15_1","unstructured":"OpenAI. 2023. GPT-4 Technical Report. arxiv:2303.08774\u00a0[cs.CL]"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3513140"},{"key":"e_1_3_2_1_17_1","unstructured":"Sida Peng Eirini Kalliamvakou Peter Cihon and Mert Demirer. 2023. The Impact of AI on Developer Productivity: Evidence from GitHub Copilot. arxiv:2302.06590\u00a0[cs.SE]"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287324.3287374"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3230977.3230981"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"Laria Reynolds and Kyle McDonell. 2021. Prompt Programming for Large Language Models: Beyond the Few-Shot Paradigm. arxiv:2102.07350\u00a0[cs.CL]","DOI":"10.1145\/3411763.3451760"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3587102.3588792"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3017680.3017781"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/990555.990598"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3570220"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569830"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3230977.3230999"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3520312.3534864"}],"event":{"name":"COMPUTE '23: 16th Annual ACM India Compute Conference","location":"Hyderabad India","acronym":"COMPUTE '23"},"container-title":["Proceedings of the 16th Annual ACM India Compute Conference"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3627217.3627235","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3627217.3627235","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T02:16:42Z","timestamp":1755915402000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3627217.3627235"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,12,9]]},"references-count":27,"alternative-id":["10.1145\/3627217.3627235","10.1145\/3627217"],"URL":"https:\/\/doi.org\/10.1145\/3627217.3627235","relation":{},"subject":[],"published":{"date-parts":[[2023,12,9]]},"assertion":[{"value":"2023-12-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}