{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T00:05:56Z","timestamp":1755907556193,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":37,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,6,14]],"date-time":"2023-06-14T00:00:00Z","timestamp":1686700800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62150410441"],"award-info":[{"award-number":["62150410441"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,6,14]]},"DOI":"10.1145\/3593434.3594236","type":"proceedings-article","created":{"date-parts":[[2023,5,30]],"date-time":"2023-05-30T12:54:01Z","timestamp":1685451241000},"page":"398-405","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["Optimized Tokenization Process for Open-Vocabulary Code Completion: An Empirical Study"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7318-8137","authenticated-orcid":false,"given":"Yasir","family":"Hussain","sequence":"first","affiliation":[{"name":"Collage of Computer Science and Technology, Nanjing University of Aeronautics and Astronautics, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6843-1892","authenticated-orcid":false,"given":"Zhiqiu","family":"Huang","sequence":"additional","affiliation":[{"name":"Collage of Computer Science and Technology, Nanjing University of Aeronautics and Astronautics, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3723-7584","authenticated-orcid":false,"given":"Yu","family":"Zhou","sequence":"additional","affiliation":[{"name":"Collage of Computer Science and Technology, Nanjing University of Aeronautics and Astronautics, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7180-8179","authenticated-orcid":false,"given":"Izhar Ahmed","family":"Khan","sequence":"additional","affiliation":[{"name":"Collage of Computer Science and Technology, Nanjing University of Aeronautics and Astronautics, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4942-9583","authenticated-orcid":false,"given":"Nasrullah","family":"Khan","sequence":"additional","affiliation":[{"name":"Comsats University, Pakistan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6643-8820","authenticated-orcid":false,"given":"Muhammad Zahid","family":"Abbas","sequence":"additional","affiliation":[{"name":"Comsats University, Pakistan"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,6,14]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Unified Pre-training for Program Understanding and Generation. arXiv preprint arXiv:2103.06333","author":"Ahmad Wasi\u00a0Uddin","year":"2021","unstructured":"Wasi\u00a0Uddin Ahmad, Saikat Chakraborty, Baishakhi Ray, and Kai-Wei Chang. 2021. Unified Pre-training for Program Understanding and Generation. arXiv preprint arXiv:2103.06333 (2021)."},{"key":"e_1_3_2_1_2_1","volume-title":"On the properties of neural machine translation: Encoder-decoder approaches. arXiv preprint arXiv:1409.1259","author":"Cho Kyunghyun","year":"2014","unstructured":"Kyunghyun Cho, Bart Van\u00a0Merri\u00ebnboer, Dzmitry Bahdanau, and Yoshua Bengio. 2014. On the properties of neural machine translation: Encoder-decoder approaches. arXiv preprint arXiv:1409.1259 (2014)."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2021.3128234"},{"key":"e_1_3_2_1_4_1","volume-title":"Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018. Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805 (2018)."},{"key":"e_1_3_2_1_5_1","volume-title":"Contrastive Learning for Source Code with Structural and Functional Properties. arXiv preprint arXiv:2110.03868","author":"Ding Yangruibo","year":"2021","unstructured":"Yangruibo Ding, Luca Buratti, Saurabh Pujar, Alessandro Morari, Baishakhi Ray, and Saikat Chakraborty. 2021. Contrastive Learning for Source Code with Structural and Functional Properties. arXiv preprint arXiv:2110.03868 (2021)."},{"key":"e_1_3_2_1_6_1","volume-title":"Finding structure in time. Cognitive science 14, 2","author":"Elman L","year":"1990","unstructured":"Jeffrey\u00a0L Elman. 1990. Finding structure in time. Cognitive science 14, 2 (1990), 179\u2013211."},{"key":"e_1_3_2_1_7_1","volume-title":"Codebert: A pre-trained model for programming and natural languages. arXiv preprint arXiv:2002.08155","author":"Feng Zhangyin","year":"2020","unstructured":"Zhangyin Feng, Daya Guo, Duyu Tang, Nan Duan, Xiaocheng Feng, Ming Gong, Linjun Shou, Bing Qin, Ting Liu, Daxin Jiang, 2020. Codebert: A pre-trained model for programming and natural languages. arXiv preprint arXiv:2002.08155 (2020)."},{"key":"e_1_3_2_1_8_1","volume-title":"Graphcodebert: Pre-training code representations with data flow. arXiv preprint arXiv:2009.08366","author":"Guo Daya","year":"2020","unstructured":"Daya Guo, Shuo Ren, Shuai Lu, Zhangyin Feng, Duyu Tang, Shujie Liu, Long Zhou, Nan Duan, Alexey Svyatkovskiy, Shengyu Fu, 2020. Graphcodebert: Pre-training code representations with data flow. arXiv preprint arXiv:2009.08366 (2020)."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.5555\/2337223.2337322"},{"key":"e_1_3_2_1_10_1","volume-title":"Long short-term memory. Neural computation 9, 8","author":"Hochreiter Sepp","year":"1997","unstructured":"Sepp Hochreiter and J\u00fcrgen Schmidhuber. 1997. Long short-term memory. Neural computation 9, 8 (1997), 1735\u20131780."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-019-09730-9"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2020.106309"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2022.111553"},{"key":"e_1_3_2_1_14_1","volume-title":"International Conference on Machine Learning. PMLR, 5110\u20135121","author":"Kanade Aditya","year":"2020","unstructured":"Aditya Kanade, Petros Maniatis, Gogul Balakrishnan, and Kensen Shi. 2020. Learning and evaluating contextual embedding of source code. In International Conference on Machine Learning. PMLR, 5110\u20135121."},{"key":"e_1_3_2_1_15_1","volume-title":"Maybe deep neural networks are the best choice for modeling source code. arXiv preprint arXiv:1903.05734","author":"Karampatsis Rafael-Michael","year":"2019","unstructured":"Rafael-Michael Karampatsis and Charles Sutton. 2019. Maybe deep neural networks are the best choice for modeling source code. arXiv preprint arXiv:1903.05734 (2019)."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/icse43902.2021.00026"},{"key":"e_1_3_2_1_17_1","volume-title":"Subword regularization: Improving neural network translation models with multiple subword candidates. arXiv preprint arXiv:1804.10959","author":"Kudo Taku","year":"2018","unstructured":"Taku Kudo. 2018. Subword regularization: Improving neural network translation models with multiple subword candidates. arXiv preprint arXiv:1804.10959 (2018)."},{"key":"e_1_3_2_1_18_1","unstructured":"Chang Liu Xin Wang Richard Shin Joseph\u00a0E Gonzalez and Dawn Song. 2016. Neural code completion. (2016)."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416591"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2018.07.006"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2012.6227205"},{"key":"e_1_3_2_1_22_1","volume-title":"A conversational paradigm for program synthesis. arXiv preprint arXiv:2203.13474","author":"Nijkamp Erik","year":"2022","unstructured":"Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, and Caiming Xiong. 2022. A conversational paradigm for program synthesis. arXiv preprint arXiv:2203.13474 (2022)."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2016.07.026"},{"key":"e_1_3_2_1_24_1","unstructured":"Alec Radford Karthik Narasimhan Tim Salimans Ilya Sutskever 2018. Improving language understanding by generative pre-training. (2018)."},{"key":"e_1_3_2_1_25_1","volume-title":"Language models are unsupervised multitask learners. OpenAI blog 1, 8","author":"Radford Alec","year":"2019","unstructured":"Alec Radford, Jeffrey Wu, Rewon Child, David Luan, Dario Amodei, Ilya Sutskever, 2019. Language models are unsupervised multitask learners. OpenAI blog 1, 8 (2019), 9."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"crossref","unstructured":"Veselin Raychev Martin Vechev Z Eth and Eran Yahav. 2014. Code completion with statistical language models. In Acm Sigplan Notices Vol.\u00a049. ACM 419\u2013428.","DOI":"10.1145\/2666356.2594321"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3417058"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2635868.2635875"},{"key":"e_1_3_2_1_29_1","unstructured":"Ashish Vaswani Noam Shazeer Niki Parmar Jakob Uszkoreit Llion Jones Aidan\u00a0N Gomez \u0141ukasz Kaiser and Illia Polosukhin. 2017. Attention is all you need. In Advances in neural information processing systems. 5998\u20136008."},{"key":"e_1_3_2_1_30_1","volume-title":"Codet5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation. arXiv preprint arXiv:2109.00859","author":"Wang Yue","year":"2021","unstructured":"Yue Wang, Weishi Wang, Shafiq Joty, and Steven\u00a0CH Hoi. 2021. Codet5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation. arXiv preprint arXiv:2109.00859 (2021)."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.5555\/2820518.2820559"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSR.2015.38"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-demos.6"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2018.03.003"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3520312.3534862"},{"key":"e_1_3_2_1_36_1","volume-title":"Recent trends in deep learning based natural language processing. ieee Computational intelligenCe magazine 13, 3","author":"Young Tom","year":"2018","unstructured":"Tom Young, Devamanyu Hazarika, Soujanya Poria, and Erik Cambria. 2018. Recent trends in deep learning based natural language processing. ieee Computational intelligenCe magazine 13, 3 (2018), 55\u201375."},{"key":"e_1_3_2_1_37_1","volume-title":"Boosting API recommendation with implicit feedback","author":"Zhou Yu","year":"2021","unstructured":"Yu Zhou, Xinying Yang, Taolue Chen, Zhiqiu Huang, Xiaoxing Ma, and Harald\u00a0C Gall. 2021. Boosting API recommendation with implicit feedback. IEEE Transactions on Software Engineering (2021)."}],"event":{"name":"EASE '23: The International Conference on Evaluation and Assessment in Software Engineering","acronym":"EASE '23","location":"Oulu Finland"},"container-title":["Proceedings of the 27th International Conference on Evaluation and Assessment in Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3593434.3594236","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3593434.3594236","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T07:09:53Z","timestamp":1755846593000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3593434.3594236"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,6,14]]},"references-count":37,"alternative-id":["10.1145\/3593434.3594236","10.1145\/3593434"],"URL":"https:\/\/doi.org\/10.1145\/3593434.3594236","relation":{},"subject":[],"published":{"date-parts":[[2023,6,14]]},"assertion":[{"value":"2023-06-14","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}