{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,21]],"date-time":"2026-04-21T15:18:02Z","timestamp":1776784682638,"version":"3.51.2"},"publisher-location":"New York, NY, USA","reference-count":73,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T00:00:00Z","timestamp":1701302400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61932021,62002125"],"award-info":[{"award-number":["61932021,62002125"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000781","name":"European Research Council","doi-asserted-by":"publisher","award":["949014"],"award-info":[{"award-number":["949014"]}],"id":[{"id":"10.13039\/501100000781","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,11,30]]},"DOI":"10.1145\/3611643.3616323","type":"proceedings-article","created":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T23:14:38Z","timestamp":1701386078000},"page":"375-387","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":19,"title":["Natural Language to Code: How Far Are We?"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1469-2063","authenticated-orcid":false,"given":"Shangwen","family":"Wang","sequence":"first","affiliation":[{"name":"National University of Defense Technology, Changsha, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7239-1819","authenticated-orcid":false,"given":"Mingyang","family":"Geng","sequence":"additional","affiliation":[{"name":"National University of Defense Technology, Changsha, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5905-4677","authenticated-orcid":false,"given":"Bo","family":"Lin","sequence":"additional","affiliation":[{"name":"National University of Defense Technology, Changsha, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5393-7858","authenticated-orcid":false,"given":"Zhensu","family":"Sun","sequence":"additional","affiliation":[{"name":"Singapore Management University, Singapore, Singapore"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5588-9618","authenticated-orcid":false,"given":"Ming","family":"Wen","sequence":"additional","affiliation":[{"name":"Huazhong University of Science and Technology, Wuhan, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8147-8126","authenticated-orcid":false,"given":"Yepang","family":"Liu","sequence":"additional","affiliation":[{"name":"Southern University of Science and Technology, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2990-1614","authenticated-orcid":false,"given":"Li","family":"Li","sequence":"additional","affiliation":[{"name":"Beihang University, Beijing, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7270-9869","authenticated-orcid":false,"given":"Tegawend\u00e9 F.","family":"Bissyand\u00e9","sequence":"additional","affiliation":[{"name":"University of Luxembourg, Luxembourg, Luxembourg"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4204-7424","authenticated-orcid":false,"given":"Xiaoguang","family":"Mao","sequence":"additional","affiliation":[{"name":"National University of Defense Technology, Changsha, China"}]}],"member":"320","published-online":{"date-parts":[[2023,11,30]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"[n.d.]. Intermediate representation. https:\/\/en.wikipedia.org\/wiki\/Intermediate_representation"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"crossref","unstructured":"Wasi Uddin Ahmad Saikat Chakraborty Baishakhi Ray and Kai-Wei Chang. 2021. Unified pre-training for program understanding and generation. arXiv preprint arXiv:2103.06333.","DOI":"10.18653\/v1\/2021.naacl-main.211"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3212695"},{"key":"e_1_3_2_2_4_1","volume-title":"Noise reduction in speech processing","author":"Benesty Jacob","unstructured":"Jacob Benesty, Jingdong Chen, Yiteng Huang, and Israel Cohen. 2009. Pearson correlation coefficient. In Noise reduction in speech processing. Springer, 1\u20134."},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICPC.2009.5090039"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753402"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/1518701.1518944"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462840"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3340458"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3540250.3549162"},{"key":"e_1_3_2_2_11_1","volume-title":"Proceedings of Machine Translation Summit IX: Papers.","author":"Coughlin Deborah","year":"2003","unstructured":"Deborah Coughlin. 2003. Correlating automated and human assessments of machine translation quality. In Proceedings of Machine Translation Summit IX: Papers."},{"key":"e_1_3_2_2_12_1","volume-title":"Code Search: A Survey of Techniques for Finding Code. arXiv preprint arXiv:2204.02765.","author":"Grazia Luca Di","year":"2022","unstructured":"Luca Di Grazia and Michael Pradel. 2022. Code Search: A Survey of Techniques for Finding Code. arXiv preprint arXiv:2204.02765."},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1068"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1882291.1882315"},{"key":"e_1_3_2_2_16_1","first-page":"23","article-title":"A new algorithm for data compression","volume":"12","author":"Gage Philip","year":"1994","unstructured":"Philip Gage. 1994. A new algorithm for data compression. C Users Journal, 12, 2 (1994), 23\u201338.","journal-title":"C Users Journal"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180167"},{"key":"e_1_3_2_2_18_1","unstructured":"Daya Guo Shuo Ren Shuai Lu Zhangyin Feng Duyu Tang Shujie Liu Long Zhou Nan Duan Alexey Svyatkovskiy and Shengyu Fu. 2021. GraphCodeBERT: Pre-training Code Representations with Data Flow. In ICLR."},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/32.54292"},{"key":"e_1_3_2_2_20_1","volume-title":"A retrieve-and-edit framework for predicting structured outputs. Advances in Neural Information Processing Systems, 31","author":"Hashimoto Tatsunori B","year":"2018","unstructured":"Tatsunori B Hashimoto, Kelvin Guu, Yonatan Oren, and Percy S Liang. 2018. A retrieve-and-edit framework for predicting structured outputs. Advances in Neural Information Processing Systems, 31 (2018)."},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1111"},{"key":"e_1_3_2_2_22_1","unstructured":"Xing Hu Qiuyuan Chen Haoye Wang Xin Xia David Lo and Thomas Zimmermann. 2021. Correlating Automated and Human Evaluation of Code Documentation Generation Quality. ACM Transactions on Software Engineering and Methodology."},{"key":"e_1_3_2_2_23_1","unstructured":"Hamel Husain Ho-Hsiang Wu Tiferet Gazit Miltiadis Allamanis and Marc Brockschmidt. 2019. Codesearchnet challenge: Evaluating the state of semantic code search. arXiv preprint arXiv:1909.09436."},{"key":"e_1_3_2_2_24_1","unstructured":"Srinivasan Iyer Ioannis Konstas Alvin Cheung and Luke Zettlemoyer. 2018. Mapping Language to Code in Programmatic Context. In EMNLP."},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2004.47"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3383458"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3540250.3549081"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3387904.3389278"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1057"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-008-0118-x"},{"key":"e_1_3_2_2_31_1","first-page":"1","article-title":"Opportunities and challenges in code search tools","volume":"54","author":"Liu Chao","year":"2021","unstructured":"Chao Liu, Xin Xia, David Lo, Cuiyun Gao, Xiaohu Yang, and John Grundy. 2021. Opportunities and challenges in code search tools. ACM Computing Surveys (CSUR), 54, 9 (2021), 1\u201340.","journal-title":"ACM Computing Surveys (CSUR)"},{"key":"e_1_3_2_2_32_1","first-page":"1","article-title":"Deep Learning Based Program Generation from Requirements Text: Are We There Yet","author":"Liu Hui","year":"2020","unstructured":"Hui Liu, Mingzhu Shen, Jiaqi Zhu, Nan Niu, Ge Li, and Lu Zhang. 2020. Deep Learning Based Program Generation from Requirements Text: Are We There Yet? IEEE Transactions on Software Engineering, 1\u20131.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_2_33_1","volume-title":"CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation. In Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 1).","author":"Lu Shuai","year":"2021","unstructured":"Shuai Lu, Daya Guo, Shuo Ren, Junjie Huang, Alexey Svyatkovskiy, Ambrosio Blanco, Colin Clement, Dawn Drain, Daxin Jiang, and Duyu Tang. 2021. CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation. In Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 1)."},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2015.42"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1214\/aoms"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00041"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/1985793.1985809"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2012.6227134"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-007-9040-x"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468564"},{"key":"e_1_3_2_2_41_1","volume-title":"Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering. 1372\u20131384","author":"Nguyen Son","unstructured":"Son Nguyen, Hung Phan, Trinh Le, and Tien N. Nguyen. 2020. Suggesting Natural Method Names to Check Name Consistencies. In Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering. 1372\u20131384."},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510096"},{"key":"e_1_3_2_2_43_1","volume-title":"Proceedings of the 40th annual meeting of the Association for Computational Linguistics. 311\u2013318","author":"Papineni Kishore","year":"2002","unstructured":"Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. Bleu: a method for automatic evaluation of machine translation. In Proceedings of the 40th annual meeting of the Association for Computational Linguistics. 311\u2013318."},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/2001420.2001445"},{"key":"e_1_3_2_2_45_1","first-page":"1","article-title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","volume":"21","author":"Raffel Colin","year":"2020","unstructured":"Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, and Peter J Liu. 2020. Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer. Journal of Machine Learning Research, 21 (2020), 1\u201367.","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_2_46_1","unstructured":"Shuo Ren Daya Guo Shuai Lu Long Zhou Shujie Liu Duyu Tang Neel Sundaresan Ming Zhou Ambrosio Blanco and Shuai Ma. 2020. Codebleu: a method for automatic evaluation of code synthesis. arXiv preprint arXiv:2009.10297."},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/2786805.2786855"},{"key":"e_1_3_2_2_48_1","first-page":"537","article-title":"A Survey of Automatic Code Generation from Natural Language","volume":"17","author":"Shin Jiho","year":"2021","unstructured":"Jiho Shin and Jaechang Nam. 2021. A Survey of Automatic Code Generation from Natural Language. Journal of Information Processing Systems, 17, 3 (2021), 537\u2013555.","journal-title":"Journal of Information Processing Systems"},{"key":"e_1_3_2_2_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3387904.3389269"},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1109\/WPC.1998.693351"},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510140"},{"key":"e_1_3_2_2_52_1","volume-title":"On the Importance of Building High-quality Training Datasets for Neural Code Search. In 2022 IEEE\/ACM 44th International Conference on Software Engineering (ICSE).","author":"Sun Zhensu","year":"2022","unstructured":"Zhensu Sun, Li Li, Yan Liu, Xiaoning Du, and Li Li. 2022. On the Importance of Building High-quality Training Datasets for Neural Code Search. In 2022 IEEE\/ACM 44th International Conference on Software Engineering (ICSE)."},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33017055"},{"key":"e_1_3_2_2_54_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i05.6430"},{"key":"e_1_3_2_2_55_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/P15-1150"},{"key":"e_1_3_2_2_56_1","volume-title":"\u0141 ukasz Kaiser, and Illia Polosukhin","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, \u0141 ukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. Advances in neural information processing systems, 30 (2017)."},{"key":"e_1_3_2_2_57_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2019.00012"},{"key":"e_1_3_2_2_58_1","doi-asserted-by":"publisher","unstructured":"Shangwen Wang. 2023. The Artifact of the ESEC\/FSE 2023 Paper Titled \"Natural Language to Code: How Far Are We?\". https:\/\/doi.org\/10.5281\/zenodo.7546358 10.5281\/zenodo.7546358","DOI":"10.5281\/zenodo.7546358"},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/3622815"},{"key":"e_1_3_2_2_60_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.685"},{"key":"e_1_3_2_2_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/337180.337615"},{"key":"e_1_3_2_2_62_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-017-9514-4"},{"key":"e_1_3_2_2_63_1","first-page":"1","article-title":"In-ide code generation from natural language: Promise and challenges","volume":"31","author":"Xu Frank F","year":"2022","unstructured":"Frank F Xu, Bogdan Vasilescu, and Graham Neubig. 2022. In-ide code generation from natural language: Promise and challenges. ACM Transactions on Software Engineering and Methodology (TOSEM), 31, 2 (2022), 1\u201347.","journal-title":"ACM Transactions on Software Engineering and Methodology (TOSEM)"},{"key":"e_1_3_2_2_64_1","volume-title":"Two-Stage Attention-Based Model for Code Search with Textual and Structural Features. In 2021 IEEE International Conference on Software Analysis, Evolution and Reengineering (SANER). 342\u2013353","author":"Xu Ling","year":"2021","unstructured":"Ling Xu, Huanhuan Yang, Chao Liu, Jianhang Shuai, Meng Yan, Yan Lei, and Zhou Xu. 2021. Two-Stage Attention-Based Model for Code Search with Textual and Structural Features. In 2021 IEEE International Conference on Software Analysis, Evolution and Reengineering (SANER). 342\u2013353."},{"key":"e_1_3_2_2_65_1","volume-title":"Jayavardhan Reddy Peddamail, and Huan Sun","author":"Yao Ziyu","year":"2019","unstructured":"Ziyu Yao, Jayavardhan Reddy Peddamail, and Huan Sun. 2019. Coacor: Code annotation for code retrieval with reinforcement learning. In The world wide web conference. 2203\u20132214."},{"key":"e_1_3_2_2_66_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1041"},{"key":"e_1_3_2_2_67_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-2002"},{"key":"e_1_3_2_2_68_1","volume-title":"CERT: Continual Pre-Training on Sketches for Library-Oriented Code Generation. arXiv preprint arXiv:2206.06888.","author":"Zan Daoguang","year":"2022","unstructured":"Daoguang Zan, Bei Chen, Dejian Yang, Zeqi Lin, Minsu Kim, Bei Guan, Yongji Wang, Weizhu Chen, and Jian-Guang Lou. 2022. CERT: Continual Pre-Training on Sketches for Library-Oriented Code Generation. arXiv preprint arXiv:2206.06888."},{"key":"e_1_3_2_2_69_1","doi-asserted-by":"crossref","unstructured":"Chen Zeng Yue Yu Shanshan Li Xin Xia Zhiming Wang Mingyang Geng Linxiao Bai Wei Dong and Xiangke Liao. 2022. deGraphCS: Embedding Variable-based Flow Graph for Neural Code Search. ACM Transactions on Software Engineering and Methodology (TOSEM).","DOI":"10.1145\/3546066"},{"key":"e_1_3_2_2_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/3533767.3534390"},{"key":"e_1_3_2_2_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/3551349.3556955"},{"key":"e_1_3_2_2_72_1","volume-title":"Reddy","author":"Zhu Ming","year":"2022","unstructured":"Ming Zhu, Aneesh Jain, Karthik Suresh, Roshan Ravindran, Sindhu Tipirneni, and Chandan K. Reddy. 2022. XLCoST: A Benchmark Dataset for Cross-lingual Code Intelligence. arxiv:2206.08474. arxiv:2206.08474"},{"key":"e_1_3_2_2_73_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416530"}],"event":{"name":"ESEC\/FSE '23: 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","location":"San Francisco CA USA","acronym":"ESEC\/FSE '23","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3611643.3616323","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3611643.3616323","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:36:11Z","timestamp":1750178171000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3611643.3616323"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,30]]},"references-count":73,"alternative-id":["10.1145\/3611643.3616323","10.1145\/3611643"],"URL":"https:\/\/doi.org\/10.1145\/3611643.3616323","relation":{},"subject":[],"published":{"date-parts":[[2023,11,30]]},"assertion":[{"value":"2023-11-30","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}