{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,19]],"date-time":"2026-02-19T15:08:00Z","timestamp":1771513680555,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":55,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,12,19]],"date-time":"2024-12-19T00:00:00Z","timestamp":1734566400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,12,19]]},"DOI":"10.1145\/3704522.3704535","type":"proceedings-article","created":{"date-parts":[[2025,1,3]],"date-time":"2025-01-03T12:29:51Z","timestamp":1735907391000},"page":"32-40","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["Assessing ChatGPT\u2019s Code Generation Capabilities with Short vs Long Context Programming Problems"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0005-9676-7779","authenticated-orcid":false,"given":"Uddip Acharjee","family":"Shuvo","sequence":"first","affiliation":[{"name":"Institute of Information Technology, University of Dhaka, Bangladesh, Dhaka, Bangladesh"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-0959-2638","authenticated-orcid":false,"given":"Sajib Acharjee","family":"Dip","sequence":"additional","affiliation":[{"name":"Computer Science and Engineering, Virginia Tech, USA, Blacksburg, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-3184-841X","authenticated-orcid":false,"given":"Nirvar Roy","family":"Vaskar","sequence":"additional","affiliation":[{"name":"American International University - Bangladesh, Dhaka, Bangladesh"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8159-6114","authenticated-orcid":false,"given":"A. B. M. Alim","family":"Al Islam","sequence":"additional","affiliation":[{"name":"Computer Science and Engineering, Bangladesh University of Engineering and Technology, Dhaka, Bangladesh"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,1,3]]},"reference":[{"key":"e_1_3_3_1_2_2","unstructured":"AI@Meta. 2024. Llama 3 Model Card. (2024). https:\/\/github.com\/meta-llama\/llama3\/blob\/main\/MODEL_CARD.md"},{"key":"e_1_3_3_1_3_2","doi-asserted-by":"crossref","unstructured":"Miltiadis Allamanis Earl\u00a0T Barr Premkumar Devanbu and Charles Sutton. 2018. A survey of machine learning for big code and naturalness. ACM Computing Surveys (CSUR) 51 4 (2018) 1\u201337.","DOI":"10.1145\/3212695"},{"key":"e_1_3_3_1_4_2","unstructured":"AI Anthropic. 2024. The claude 3 model family: Opus sonnet haiku. Claude-3 Model Card 1 (2024)."},{"key":"e_1_3_3_1_5_2","unstructured":"Yannick Assogba and Donghao Ren. 2024. Evaluating Long Range Dependency Handling in Code Generation Models using Multi-Step Key Retrieval. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.21049 (2024)."},{"key":"e_1_3_3_1_6_2","unstructured":"Ben Athiwaratkun Sanjay\u00a0Krishna Gouda Zijian Wang Xiaopeng Li Yuchen Tian Ming Tan Wasi\u00a0Uddin Ahmad Shiqi Wang Qing Sun Mingyue Shang et\u00a0al. 2022. Multi-lingual evaluation of code generation models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2210.14868 (2022)."},{"key":"e_1_3_3_1_7_2","unstructured":"Somnath Banerjee Avik Dutta Sayan Layek Amruit Sahoo Sam\u00a0Conrad Joyce and Rima Hazra. 2023. Redefining Developer Assistance: Through Large Language Models in Software Ecosystem. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2312.05626 (2023)."},{"key":"e_1_3_3_1_8_2","unstructured":"Robin Beer Alexander Feix Tim Guttzeit Tamara Muras Vincent M\u00fcller Maurice Rauscher Florian Sch\u00e4ffler and Welf L\u00f6we. 2024. Examination of Code generated by Large Language Models. arxiv:https:\/\/arXiv.org\/abs\/2408.16601\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2408.16601"},{"key":"e_1_3_3_1_9_2","unstructured":"Gonzalo Benegas Chengzhong Ye Carlos Albors Jianan\u00a0Canal Li and Yun\u00a0S Song. 2024. Genomic language models: opportunities and challenges. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.11435 (2024)."},{"key":"e_1_3_3_1_10_2","unstructured":"Zhenyu Bi Sajib\u00a0Acharjee Dip Daniel Hajialigol Sindhura Kommu Hanwen Liu Meng Lu and Xuan Wang. 2024. AI for Biomedicine in the Era of Large Language Models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2403.15673 (2024)."},{"key":"e_1_3_3_1_11_2","unstructured":"Tom\u00a0B Brown. 2020. Language models are few-shot learners. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2005.14165 (2020)."},{"key":"e_1_3_3_1_12_2","doi-asserted-by":"crossref","unstructured":"Alessio Bucaioni Hampus Ekedahl Vilma Helander and Phuong\u00a0T Nguyen. 2024. Programming with ChatGPT: How far can we go? Machine Learning with Applications 15 (2024) 100526.","DOI":"10.1016\/j.mlwa.2024.100526"},{"key":"e_1_3_3_1_13_2","unstructured":"Codeforces [n. d.]. Codeforces. https:\/\/codeforces.com. Accessed: 15-Aug-2023."},{"key":"e_1_3_3_1_14_2","doi-asserted-by":"crossref","unstructured":"Carlos Eduardo\u00a0Andino Coello Mohammed\u00a0Nazeh Alimam and Rand Kouatly. 2024. Effectiveness of ChatGPT in coding: a comparative analysis of popular large language models. Digital 4 1 (2024) 114\u2013125.","DOI":"10.3390\/digital4010005"},{"key":"e_1_3_3_1_15_2","doi-asserted-by":"crossref","unstructured":"Domenico Cotroneo Alessio Foggia Cristina Improta Pietro Liguori and Roberto Natella. 2024. Automating the correctness assessment of AI-generated code for security contexts. Journal of Systems and Software (2024) 112113.","DOI":"10.1016\/j.jss.2024.112113"},{"key":"e_1_3_3_1_16_2","unstructured":"Jacob Devlin. 2018. Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1810.04805 (2018)."},{"key":"e_1_3_3_1_17_2","doi-asserted-by":"publisher","DOI":"10.1609\/aaaiss.v4i1.31800"},{"key":"e_1_3_3_1_18_2","unstructured":"Sajib\u00a0Acharjee Dip Uddip\u00a0Acharjee Shuvo Tran Chau Haoqiu Song Petra Choi Xuan Wang and Liqing Zhang. 2024. PathoLM: Identifying pathogenicity from the DNA sequence through the Genome Foundation Model. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2406.13133 (2024)."},{"key":"e_1_3_3_1_19_2","unstructured":"Shihan Dou Haoxiang Jia Shenxi Wu Huiyuan Zheng Weikang Zhou Muling Wu Mingxu Chai Jessica Fan Caishuang Huang Yunbo Tao et\u00a0al. 2024. What\u2019s Wrong with Your Code Generated by Large Language Models? An Extensive Study. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.06153 (2024)."},{"key":"e_1_3_3_1_20_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-FoSE59343.2023.00008"},{"key":"e_1_3_3_1_21_2","doi-asserted-by":"crossref","unstructured":"Stefano Filippi and Barbara Motyl. 2024. Large Language Models (LLMs) in Engineering Education: A Systematic Review and Suggestions for Practical Adoption. Information 15 6 (2024) 345.","DOI":"10.3390\/info15060345"},{"key":"e_1_3_3_1_22_2","unstructured":"HackerRank [n. d.]. HackerRank. https:\/\/www.hackerrank.com. Accessed: 15-Aug-2023."},{"key":"e_1_3_3_1_23_2","unstructured":"Srinivasan Iyer Ioannis Konstas Alvin Cheung and Luke Zettlemoyer. 2018. Mapping language to code in programmatic context. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1808.09588 (2018)."},{"key":"e_1_3_3_1_24_2","unstructured":"Juyong Jiang Fan Wang Jiasi Shen Sungju Kim and Sunghun Kim. 2024. A Survey on Large Language Models for Code Generation. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2406.00515 (2024)."},{"key":"e_1_3_3_1_25_2","doi-asserted-by":"publisher","DOI":"10.1145\/3643991.3645074"},{"key":"e_1_3_3_1_26_2","doi-asserted-by":"crossref","unstructured":"Ranim Khojah Mazen Mohamad Philipp Leitner and Francisco\u00a0Gomes de Oliveira\u00a0Neto. 2024. Beyond code generation: An observational study of chatgpt usage in software engineering practice. Proceedings of the ACM on Software Engineering 1 FSE (2024) 1819\u20131840.","DOI":"10.1145\/3660788"},{"key":"e_1_3_3_1_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/3387904.3389268"},{"key":"e_1_3_3_1_28_2","unstructured":"LeetCode [n. d.]. LeetCode. https:\/\/leetcode.com. Accessed: 15-Aug-2023."},{"key":"e_1_3_3_1_29_2","unstructured":"Tianle Li Ge Zhang Quy\u00a0Duc Do Xiang Yue and Wenhu Chen. 2024. Long-context llms struggle with long in-context learning. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2404.02060 (2024)."},{"key":"e_1_3_3_1_30_2","unstructured":"Jiawei Liu Chunqiu\u00a0Steven Xia Yuyao Wang and Lingming Zhang. 2024. Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation. Advances in Neural Information Processing Systems 36 (2024)."},{"key":"e_1_3_3_1_31_2","unstructured":"Stephen MacNeil Andrew Tran Juho Leinonen Paul Denny Joanne Kim Arto Hellas Seth Bernstein and Sami Sarsa. 2022. Automatically generating cs learning materials with large language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2212.05113 (2022)."},{"key":"e_1_3_3_1_32_2","doi-asserted-by":"publisher","DOI":"10.1109\/AITest62860.2024.00022"},{"key":"e_1_3_3_1_33_2","doi-asserted-by":"crossref","unstructured":"Ansong Ni Pengcheng Yin Yilun Zhao Martin Riddell Troy Feng Rui Shen Stephen Yin Ye Liu Semih Yavuz Caiming Xiong et\u00a0al. 2024. L2ceval: Evaluating language-to-code generation capabilities of large language models. Transactions of the Association for Computational Linguistics 12 (2024) 1311\u20131329.","DOI":"10.1162\/tacl_a_00705"},{"key":"e_1_3_3_1_34_2","doi-asserted-by":"publisher","DOI":"10.1145\/3650105.3652295"},{"key":"e_1_3_3_1_35_2","unstructured":"Debalina\u00a0Ghosh Paul Hong Zhu and Ian Bayley. 2024. Benchmarks and Metrics for Evaluations of Code Generation: A Critical Review. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2406.12655 (2024)."},{"key":"e_1_3_3_1_36_2","doi-asserted-by":"publisher","DOI":"10.1145\/3643795.3648379"},{"key":"e_1_3_3_1_37_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.153"},{"key":"e_1_3_3_1_38_2","doi-asserted-by":"publisher","DOI":"10.1145\/3581641.3584037"},{"key":"e_1_3_3_1_39_2","doi-asserted-by":"crossref","unstructured":"Zolt\u00e1n S\u00e1godi Istv\u00e1n Siket and Rudolf Ferenc. 2024. Methodology for Code Synthesis Evaluation of LLMs Presented by a Case Study of ChatGPT and Copilot. IEEE Access (2024).","DOI":"10.1109\/ACCESS.2024.3403858"},{"key":"e_1_3_3_1_40_2","unstructured":"Ensheng Shi Fengji Zhang Yanlin Wang Bei Chen Lun Du Hongyu Zhang Shi Han Dongmei Zhang and Hongbin Sun. 2023. SoTaNa: The Open-Source Software Development Assistant. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2308.13416 (2023)."},{"key":"e_1_3_3_1_41_2","unstructured":"Mohammed\u00a0Latif Siddiq Simantika Dristi Joy Saha and Joanna\u00a0CS Santos. [n. d.]. The Fault in our Stars: Quality Assessment of Code Generation Benchmarks. ([n. d.])."},{"key":"e_1_3_3_1_42_2","doi-asserted-by":"publisher","DOI":"10.1145\/3643991.3645071"},{"key":"e_1_3_3_1_43_2","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3417058"},{"key":"e_1_3_3_1_44_2","unstructured":"Florian Tambon Arghavan\u00a0Moradi Dakhel Amin Nikanjam Foutse Khomh Michel\u00a0C Desmarais and Giuliano Antoniol. 2024. Bugs in large language models generated code. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2403.08937 (2024)."},{"key":"e_1_3_3_1_45_2","unstructured":"Ben\u00a0Arie Tanay Lexy Arinze Siddhant\u00a0S Joshi Kirsten\u00a0A Davis and James\u00a0C Davis. 2024. An Exploratory Study on Upper-Level Computing Students\u2019 Use of Large Language Models as Tools in a Semester-Long Project. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2403.18679 (2024)."},{"key":"e_1_3_3_1_46_2","unstructured":"CodeGemma Team Heri Zhao Jeffrey Hui Joshua Howland Nam Nguyen Siqi Zuo Andrea Hu Christopher\u00a0A Choquette-Choo Jingyue Shen Joe Kelley et\u00a0al. 2024. Codegemma: Open code models based on gemma. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2406.11409 (2024)."},{"key":"e_1_3_3_1_47_2","unstructured":"Hugo Touvron Thibaut Lavril Gautier Izacard Xavier Martinet Marie-Anne Lachaux Timoth\u00e9e Lacroix Baptiste Rozi\u00e8re Naman Goyal Eric Hambro Faisal Azhar et\u00a0al. 2023. Llama: Open and efficient foundation language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2302.13971 (2023)."},{"key":"e_1_3_3_1_48_2","unstructured":"UVa Online Judge [n. d.]. UVa Online Judge. https:\/\/onlinejudge.org. Accessed: 15-Aug-2023."},{"key":"e_1_3_3_1_49_2","unstructured":"Jiayin Wang Weizhi Ma Peijie Sun Min Zhang and Jian-Yun Nie. 2024. Understanding User Experience in Large Language Model Interactions. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2401.08329 (2024)."},{"key":"e_1_3_3_1_50_2","unstructured":"Yanlin Wang Tianyue Jiang Mingwei Liu Jiachi Chen and Zibin Zheng. 2024. Beyond Functional Correctness: Investigating Coding Style Inconsistencies in Large Language Models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.00456 (2024)."},{"key":"e_1_3_3_1_51_2","unstructured":"Jason Wei Xuezhi Wang Dale Schuurmans Maarten Bosma Fei Xia Ed Chi Quoc\u00a0V Le Denny Zhou et\u00a0al. 2022. Chain-of-thought prompting elicits reasoning in large language models. Advances in neural information processing systems 35 (2022) 24824\u201324837."},{"key":"e_1_3_3_1_52_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1447"},{"key":"e_1_3_3_1_53_2","unstructured":"Daoguang Zan Bei Chen Fengji Zhang Dianjie Lu Bingchao Wu Bei Guan Yongji Wang and Jian-Guang Lou. 2022. Large language models meet nl2code: A survey. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2212.09420 (2022)."},{"key":"e_1_3_3_1_54_2","first-page":"11328","volume-title":"International conference on machine learning","author":"Zhang Jingqing","year":"2020","unstructured":"Jingqing Zhang, Yao Zhao, Mohammad Saleh, and Peter Liu. 2020. Pegasus: Pre-training with extracted gap-sentences for abstractive summarization. In International conference on machine learning. PMLR, 11328\u201311339."},{"key":"e_1_3_3_1_55_2","unstructured":"Pei Zhou Jay Pujara Xiang Ren Xinyun Chen Heng-Tze Cheng Quoc\u00a0V Le Ed\u00a0H Chi Denny Zhou Swaroop Mishra and Huaixiu\u00a0Steven Zheng. 2024. Self-discover: Large language models self-compose reasoning structures. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2402.03620 (2024)."},{"key":"e_1_3_3_1_56_2","unstructured":"Shuang Zhou Zidu Xu Mian Zhang Chunpu Xu Yawen Guo Zaifu Zhan Sirui Ding Jiashuo Wang Kaishuai Xu Yi Fang et\u00a0al. 2024. Large language models for disease diagnosis: A scoping review. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2409.00097 (2024)."}],"event":{"name":"NSysS '24: 11th International Conference on Networking, Systems, and Security","location":"Khulna Karak Bangladesh","acronym":"NSysS '24"},"container-title":["Proceedings of the 11th International Conference on Networking, Systems, and Security"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3704522.3704535","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3704522.3704535","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:18:16Z","timestamp":1750295896000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3704522.3704535"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,12,19]]},"references-count":55,"alternative-id":["10.1145\/3704522.3704535","10.1145\/3704522"],"URL":"https:\/\/doi.org\/10.1145\/3704522.3704535","relation":{},"subject":[],"published":{"date-parts":[[2024,12,19]]},"assertion":[{"value":"2025-01-03","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}