{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,14]],"date-time":"2026-03-14T10:42:34Z","timestamp":1773484954885,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":70,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T00:00:00Z","timestamp":1701302400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62161146003"],"award-info":[{"award-number":["62161146003"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Tencent Foundation","award":["XPLORER PRIZE"],"award-info":[{"award-number":["XPLORER PRIZE"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,11,30]]},"DOI":"10.1145\/3611643.3616280","type":"proceedings-article","created":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T23:14:38Z","timestamp":1701386078000},"page":"1294-1306","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":20,"title":["How Practitioners Expect Code Completion?"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-3935-7328","authenticated-orcid":false,"given":"Chaozheng","family":"Wang","sequence":"first","affiliation":[{"name":"Chinese University of Hong Kong, Hong Kong, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-7087-4642","authenticated-orcid":false,"given":"Junhao","family":"Hu","sequence":"additional","affiliation":[{"name":"Peking University, Beijing, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4774-2434","authenticated-orcid":false,"given":"Cuiyun","family":"Gao","sequence":"additional","affiliation":[{"name":"Chinese University of Hong Kong, Hong Kong, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-1523-7539","authenticated-orcid":false,"given":"Yu","family":"Jin","sequence":"additional","affiliation":[{"name":"Tencent, Guangzhou, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6731-216X","authenticated-orcid":false,"given":"Tao","family":"Xie","sequence":"additional","affiliation":[{"name":"Peking University, Beijing, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-0655-9398","authenticated-orcid":false,"given":"Hailiang","family":"Huang","sequence":"additional","affiliation":[{"name":"Tencent, Guangzhou, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-8271-2441","authenticated-orcid":false,"given":"Zhenyu","family":"Lei","sequence":"additional","affiliation":[{"name":"Tencent, Guangzhou, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-7060-4109","authenticated-orcid":false,"given":"Yuetang","family":"Deng","sequence":"additional","affiliation":[{"name":"Tencent, Guangzhou, China"}]}],"member":"320","published-online":{"date-parts":[[2023,11,30]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"n.d.. aiXcoder. https:\/\/aixcoder.com\/"},{"key":"e_1_3_2_2_2_1","unstructured":"n.d.. Clangd. https:\/\/clangd.llvm.org\/"},{"key":"e_1_3_2_2_3_1","unstructured":"n.d.. Copilot. https:\/\/github.com\/features\/copilot\/"},{"key":"e_1_3_2_2_4_1","unstructured":"n.d.. Google Forms. https:\/\/docs.google.com\/forms"},{"key":"e_1_3_2_2_5_1","unstructured":"n.d.. NVivo. https:\/\/www.qsrinternational.com\/nvivo-qualitative-data-analysis-software\/home"},{"key":"e_1_3_2_2_6_1","unstructured":"n.d.. Pycharm. https:\/\/www.jetbrains.com\/pycharm\/"},{"key":"e_1_3_2_2_7_1","unstructured":"n.d.. TabNine. https:\/\/www.tabnine.com\/"},{"key":"e_1_3_2_2_8_1","unstructured":"n.d.. Tencent Questionnair. https:\/\/wj.qq.com\/"},{"key":"e_1_3_2_2_9_1","unstructured":"n.d.. Visual Studio Code. https:\/\/code.visualstudio.com\/"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/SANER.2016.39"},{"key":"e_1_3_2_2_11_1","volume-title":"Decision-Making and Collaboration: 19th International Conference, HCI International","author":"Arrebola Fabio Villamarin","year":"2017","unstructured":"Fabio Villamarin Arrebola and Plinio Thomaz Aquino Junior. 2017. On source code completion assistants and the need of a context-aware approach. In Human Interface and the Management of Information: Supporting Learning, Decision-Making and Collaboration: 19th International Conference, HCI International 2017, Vancouver, BC, Canada, July 9\u201314, Part II 19. 191\u2013201."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP52600.2021.00022"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3540250.3558968"},{"key":"e_1_3_2_2_14_1","volume-title":"33nd International Conference on Machine Learning, ICML 2016, New York City, NY, USA, June 19-24, 2016 (JMLR Workshop and Conference Proceedings","volume":"2942","author":"Bielik Pavol","unstructured":"Pavol Bielik, Veselin Raychev, and Martin T. Vechev. 2016. PHOG: probabilistic model for code. In 33nd International Conference on Machine Learning, ICML 2016, New York City, NY, USA, June 19-24, 2016 (JMLR Workshop and Conference Proceedings, Vol. 48). 2933\u20132942."},{"key":"e_1_3_2_2_15_1","volume-title":"Advances in Neural Information Processing Systems 33: Annual Conference on Neural Information Processing Systems 2020","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, and Amanda Askell. 2020. Language models are few-shot learners. Advances in Neural Information Processing Systems 33: Annual Conference on Neural Information Processing Systems 2020, NeurIPS 2020, December 6-12, 2020, virtual."},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/1595696.1595728"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSR52588.2021.00024"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.387"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2021.106690"},{"key":"e_1_3_2_2_20_1","volume-title":"2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2019","volume":"1","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. BERT: pre-training of deep bidirectional transformers for language understanding. In 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2019, Minneapolis, MN, USA, June 2-7, 2019, Volume 1 (Long and Short Papers). 4171\u20134186."},{"key":"e_1_3_2_2_21_1","volume-title":"International Conference on Learning Representations.","author":"Guo Daya","year":"2021","unstructured":"Daya Guo, Alexey Svyatkovskiy, Jian Yin, Nan Duan, Marc Brockschmidt, and Miltiadis Allamanis. 2021. Learning to complete code with sketches. In International Conference on Learning Representations."},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2491956.2462192"},{"key":"e_1_3_2_2_23_1","volume-title":"2017 11th Joint Meeting on Foundations of Software Engineering, ESEC\/FSE 2017","author":"Vincent","year":"2017","unstructured":"Vincent J. Hellendoorn and Premkumar T. Devanbu. 2017. Are deep neural networks the best choice for modeling source code? In 2017 11th Joint Meeting on Foundations of Software Engineering, ESEC\/FSE 2017, Paderborn, Germany, September 4-8, 2017. 763\u2013773."},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00101"},{"key":"e_1_3_2_2_25_1","volume-title":"34th International Conference on Software Engineering, ICSE 2012","author":"Hindle Abram","year":"2012","unstructured":"Abram Hindle, Earl T. Barr, Zhendong Su, Mark Gabel, and Premkumar T. Devanbu. 2012. On the naturalness of software. In 34th International Conference on Software Engineering, ICSE 2012, June 2-9, 2012, Zurich, Switzerland. 837\u2013847."},{"key":"e_1_3_2_2_26_1","volume-title":"2nd International Workshop on Recommendation Systems for Software Engineering, RSSE 2010","author":"Hou Daqing","year":"2010","unstructured":"Daqing Hou and David M. Pletcher. 2010. Towards a better code completion system by API grouping, filtering, and popularity-based ranking. In 2nd International Workshop on Recommendation Systems for Software Engineering, RSSE 2010, Cape Town, South Africa, May 4, 2010. 26\u201330."},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510152"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2020.106309"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2022.111553"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510172"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3526113.3545659"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2014.2318734"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00026"},{"key":"e_1_3_2_2_34_1","volume-title":"25th International Symposium on Software Testing and Analysis, ISSTA 2016","author":"Kochhar Pavneet Singh","year":"2016","unstructured":"Pavneet Singh Kochhar, Xin Xia, David Lo, and Shanping Li. 2016. Practitioners\u2019 expectations on automated fault localization. In 25th International Symposium on Software Testing and Analysis, ISSTA 2016, Saarbr\u00fccken, Germany, July 18-20, 2016. 165\u2013176."},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME52107.2021.00024"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2018\/578"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3387904.3389261"},{"key":"e_1_3_2_2_38_1","volume-title":"A unified multi-task learning model for AST-level and token-level code completion. Empir. Softw. Eng., 27, 91","author":"Liu Fang","year":"2022","unstructured":"Fang Liu, Ge Li, Bolin Wei, Xin Xia, Zhiyi Fu, and Zhi Jin. 2022. A unified multi-task learning model for AST-level and token-level code completion. Empir. Softw. Eng., 27, 91 (2022)."},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416591"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2020.110547"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.431"},{"key":"e_1_3_2_2_42_1","volume-title":"NeurIPS Datasets and Benchmarks 2021","author":"Lu Shuai","year":"2021","unstructured":"Shuai Lu, Daya Guo, Shuo Ren, Junjie Huang, Alexey Svyatkovskiy, Ambrosio Blanco, Colin B. Clement, Dawn Drain, Daxin Jiang, Duyu Tang, Ge Li, Lidong Zhou, Linjun Shou, Long Zhou, Michele Tufano, Ming Gong, Ming Zhou, Nan Duan, Neel Sundaresan, Shao Kun Deng, Shengyu Fu, and Shujie Liu. 2021. CodeXGLUE: a machine learning benchmark dataset for code understanding and generation. In Neural Information Processing Systems Track on Datasets and Benchmarks 1, NeurIPS Datasets and Benchmarks 2021, December 2021, virtual."},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/1065010.1065018"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580940"},{"key":"e_1_3_2_2_45_1","volume-title":"37th IEEE\/ACM International Conference on Software Engineering, ICSE 2015","volume":"1","author":"Nguyen Anh Tuan","year":"2015","unstructured":"Anh Tuan Nguyen and Tien N. Nguyen. 2015. Graph-based statistical language model for code. In 37th IEEE\/ACM International Conference on Software Engineering, ICSE 2015, Florence, Italy, May 16-24, 2015, Volume 1. 858\u2013868."},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3551349.3559544"},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2019.00072"},{"key":"e_1_3_2_2_48_1","volume-title":"Joint Meeting of the European Software Engineering Conference and the ACM SIGSOFT Symposium on the Foundations of Software Engineering, ESEC\/FSE 2013","author":"Nguyen Tung Thanh","year":"2013","unstructured":"Tung Thanh Nguyen, Anh Tuan Nguyen, Hoan Anh Nguyen, and Tien N. Nguyen. 2013. A statistical semantic language model for source code. In Joint Meeting of the European Software Engineering Conference and the ACM SIGSOFT Symposium on the Foundations of Software Engineering, ESEC\/FSE 2013, Saint Petersburg, Russian Federation, August 18-26, 2013. 532\u2013542."},{"key":"e_1_3_2_2_49_1","volume-title":"40th Annual Meeting of the Association for Computational Linguistics, ACL","author":"Papineni Kishore","year":"2002","unstructured":"Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. Bleu: a method for automatic evaluation of machine translation. In 40th Annual Meeting of the Association for Computational Linguistics, ACL 2002, Philadelphia, PA, USA. 311\u2013318."},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3527329"},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/2254064.2254098"},{"key":"e_1_3_2_2_52_1","volume-title":"Software Engineering","author":"Proksch Sebastian","year":"2016","unstructured":"Sebastian Proksch, Johannes Lerch, and Mira Mezini. 2016. Intelligent code completion with Bayesian networks. In Software Engineering 2016, Fachtagung des GI-Fachbereichs Softwaretechnik, 23.-26. Februar 2016, Wien, \u00d6sterreich (LNI, Vol. P-252). 25\u201326."},{"key":"e_1_3_2_2_53_1","volume-title":"Language models are unsupervised multitask learners. OpenAI blog, 1, 8","author":"Radford Alec","year":"2019","unstructured":"Alec Radford, Jeffrey Wu, Rewon Child, David Luan, Dario Amodei, and Ilya Sutskever. 2019. Language models are unsupervised multitask learners. OpenAI blog, 1, 8 (2019), 9."},{"key":"e_1_3_2_2_54_1","volume-title":"2016 ACM SIGPLAN International Conference on Object-Oriented Programming, Systems, Languages, and Applications, OOPSLA 2016, part of SPLASH 2016","author":"Raychev Veselin","year":"2016","unstructured":"Veselin Raychev, Pavol Bielik, and Martin T. Vechev. 2016. Probabilistic model for code with decision trees. In 2016 ACM SIGPLAN International Conference on Object-Oriented Programming, Systems, Languages, and Applications, OOPSLA 2016, part of SPLASH 2016, Amsterdam, The Netherlands, October 30 - November 4, 2016. 731\u2013747."},{"key":"e_1_3_2_2_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/2594291.2594321"},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2008.42"},{"key":"e_1_3_2_2_57_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2016.12.036"},{"key":"e_1_3_2_2_58_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2016.06.063"},{"key":"e_1_3_2_2_59_1","volume-title":"Fundamentals of recurrent neural network (RNN) and long short-term memory (LSTM) network. CoRR, abs\/1808.03314","author":"Sherstinsky Alex","year":"2018","unstructured":"Alex Sherstinsky. 2018. Fundamentals of recurrent neural network (RNN) and long short-term memory (LSTM) network. CoRR, abs\/1808.03314 (2018)."},{"key":"e_1_3_2_2_60_1","volume-title":"28th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, ESEC\/FSE 2020","author":"Svyatkovskiy Alexey","year":"2020","unstructured":"Alexey Svyatkovskiy, Shao Kun Deng, Shengyu Fu, and Neel Sundaresan. 2020. IntelliCode compose: code generation using transformer. In ESEC\/FSE \u201920: 28th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, ESEC\/FSE 2020, Virtual Event, USA, November 8-13, 2020. 1433\u20131443."},{"key":"e_1_3_2_2_61_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSR52588.2021.00045"},{"key":"e_1_3_2_2_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330699"},{"key":"e_1_3_2_2_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/1321631.1321663"},{"key":"e_1_3_2_2_64_1","volume-title":"22nd ACM SIGSOFT International Symposium on Foundations of Software Engineering, ESEC\/FSE 2014","author":"Tu Zhaopeng","year":"2014","unstructured":"Zhaopeng Tu, Zhendong Su, and Premkumar T. Devanbu. 2014. On the localness of software. In 22nd ACM SIGSOFT International Symposium on Foundations of Software Engineering, ESEC\/FSE 2014, Hong Kong, China, November 16 - 22, 2014. 269\u2013280."},{"key":"e_1_3_2_2_65_1","volume-title":"CHI 2022: CHI Conference on Human Factors in Computing Systems","author":"Vaithilingam Priyan","year":"2022","unstructured":"Priyan Vaithilingam, Tianyi Zhang, and Elena L. Glassman. 2022. Expectation vs. experience: evaluating the usability of code generation tools powered by large language models. In CHI 2022: CHI Conference on Human Factors in Computing Systems, New Orleans, LA, USA, 29 April 2022 - 5 May 2022, Extended Abstracts. 332:1\u2013332:7."},{"key":"e_1_3_2_2_66_1","volume-title":"Advances in Neural Information Processing Systems 30: Annual Conference on Neural Information Processing Systems NeurIPS","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. In Advances in Neural Information Processing Systems 30: Annual Conference on Neural Information Processing Systems NeurIPS 2017, December 4-9, Long Beach, CA, USA. 5998\u20136008."},{"key":"e_1_3_2_2_67_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i16.17650"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00025"},{"key":"e_1_3_2_2_69_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2017.8115678"},{"key":"e_1_3_2_2_70_1","volume-title":"USA","author":"Ziegler Albert","year":"2022","unstructured":"Albert Ziegler, Eirini Kalliamvakou, X. Alice Li, Andrew Rice, Devon Rifkin, Shawn Simister, Ganesh Sittampalam, and Edward Aftandilian. 2022. Productivity assessment of neural code completion. In MAPS@PLDI 2022: 6th ACM SIGPLAN International Symposium on Machine Programming, San Diego, CA, USA, 13 June 2022. 21\u201329."}],"event":{"name":"ESEC\/FSE '23: 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","location":"San Francisco CA USA","acronym":"ESEC\/FSE '23","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3611643.3616280","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3611643.3616280","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:36:03Z","timestamp":1750178163000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3611643.3616280"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,30]]},"references-count":70,"alternative-id":["10.1145\/3611643.3616280","10.1145\/3611643"],"URL":"https:\/\/doi.org\/10.1145\/3611643.3616280","relation":{},"subject":[],"published":{"date-parts":[[2023,11,30]]},"assertion":[{"value":"2023-11-30","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}