{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,8]],"date-time":"2026-07-08T23:39:36Z","timestamp":1783553976766,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":54,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,10,10]],"date-time":"2022-10-10T00:00:00Z","timestamp":1665360000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100000781","name":"European Research Council","doi-asserted-by":"publisher","award":["851895"],"award-info":[{"award-number":["851895"]}],"id":[{"id":"10.13039\/501100000781","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,10,10]]},"DOI":"10.1145\/3551349.3556903","type":"proceedings-article","created":{"date-parts":[[2023,1,5]],"date-time":"2023-01-05T20:43:54Z","timestamp":1672951434000},"page":"1-12","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":39,"title":["CrystalBLEU: Precisely and Efficiently Measuring the Similarity of Code"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9763-8147","authenticated-orcid":false,"given":"Aryaz","family":"Eghbali","sequence":"first","affiliation":[{"name":"University of Stuttgart, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1623-498X","authenticated-orcid":false,"given":"Michael","family":"Pradel","sequence":"additional","affiliation":[{"name":"University of Stuttgart, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,1,5]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3212695"},{"key":"e_1_3_2_1_2_1","volume-title":"7th International Conference on Learning Representations, ICLR 2019","author":"Alon Uri","year":"2019","unstructured":"Uri Alon, Shaked Brody, Omer Levy, and Eran Yahav. 2019. code2seq: Generating Sequences from Structured Representations of Code. In 7th International Conference on Learning Representations, ICLR 2019, New Orleans, LA, USA, May 6-9, 2019. https:\/\/openreview.net\/forum?id=H1gKYo09tX"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"crossref","unstructured":"Uri Alon Meital Zilberstein Omer Levy and Eran Yahav. 2018. A General Path-Based Representation for Predicting Program Properties. In PLDI.","DOI":"10.1145\/3192366.3192412"},{"key":"e_1_3_2_1_4_1","unstructured":"Gareth\u00a0Ari Aye and Gail\u00a0E. Kaiser. 2020. Sequence Model Design for Code Completion in the Modern IDE. CoRR abs\/2004.05249(2020). arxiv:2004.05249https:\/\/arxiv.org\/abs\/2004.05249"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.3115\/1218955.1219034"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","unstructured":"Patrick Barei\u00df Beatriz Souza Marcelo d\u2019Amorim and Michael Pradel. 2022. Code Generation Tools (Almost) for Free? A Study of Few-Shot Pre-Trained Language Models on Code. CoRR abs\/2206.01335(2022). https:\/\/doi.org\/10.48550\/arXiv.2206.01335 arXiv:2206.01335","DOI":"10.48550\/arXiv.2206.01335"},{"key":"e_1_3_2_1_7_1","volume-title":"Re-evaluation the Role of Bleu in Machine Translation Research. In EACL 2006, 11st Conference of the European Chapter of the Association for Computational Linguistics, Proceedings of the Conference","author":"Callison-Burch Chris","year":"2006","unstructured":"Chris Callison-Burch, Miles Osborne, and Philipp Koehn. 2006. Re-evaluation the Role of Bleu in Machine Translation Research. In EACL 2006, 11st Conference of the European Chapter of the Association for Computational Linguistics, Proceedings of the Conference, April 3-7, 2006, Trento, Italy, Diana McCarthy and Shuly Wintner (Eds.). The Association for Computer Linguistics. https:\/\/www.aclweb.org\/anthology\/E06-1032\/"},{"key":"e_1_3_2_1_8_1","unstructured":"Saikat Chakraborty Miltiadis Allamanis and Baishakhi Ray. 2018. Tree2Tree Neural Translation Model for Learning Source Code Changes. CoRR abs\/1810.00314(2018). arxiv:1810.00314http:\/\/arxiv.org\/abs\/1810.00314"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/W14-3346"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180240"},{"key":"e_1_3_2_1_11_1","volume-title":"SequenceR: Sequence-to-Sequence Learning for End-to-End Program Repair","author":"Chen Zimin","year":"2019","unstructured":"Zimin Chen, Steve Kommrusch, Michele Tufano, Louis-No\u00ebl Pouchet, Denys Poshyvanyk, and Martin Monperrus. 2019. SequenceR: Sequence-to-Sequence Learning for End-to-End Program Repair. IEEE TSE (2019)."},{"key":"e_1_3_2_1_12_1","volume-title":"8th International Conference on Learning Representations, ICLR 2020","author":"Dinella Elizabeth","year":"2020","unstructured":"Elizabeth Dinella, Hanjun Dai, Ziyang Li, Mayur Naik, Le Song, and Ke Wang. 2020. Hoppity: Learning Graph Transformations to Detect and Fix Bugs in Programs. In 8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, April 26-30, 2020. OpenReview.net. https:\/\/openreview.net\/forum?id=SJeqs6EFvB"},{"key":"e_1_3_2_1_13_1","unstructured":"Yangruibo Ding Baishakhi Ray Premkumar Devanbu and Vincent\u00a0J Hellendoorn. 2020. Patching as Translation: the Data and the Metaphor. arXiv preprint arXiv:2008.10707(2020)."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D15-1013"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","unstructured":"Xiaodong Gu Hongyu Zhang Dongmei Zhang and Sunghun Kim. 2016. Deep API learning. In FSE. 631\u2013642. https:\/\/doi.org\/10.1145\/2950290.2950334","DOI":"10.1145\/2950290.2950334"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.10742"},{"key":"e_1_3_2_1_17_1","volume-title":"Advances in Neural Information Processing Systems 31: Annual Conference on Neural Information Processing Systems 2018","author":"Harer Jacob","year":"2018","unstructured":"Jacob Harer, Onur Ozdemir, Tomo Lazovich, Christopher\u00a0P. Reale, Rebecca\u00a0L. Russell, Louis\u00a0Y. Kim, and Sang\u00a0Peter Chin. 2018. Learning to Repair Software Vulnerabilities with Generative Adversarial Networks. In Advances in Neural Information Processing Systems 31: Annual Conference on Neural Information Processing Systems 2018, NeurIPS 2018, 3-8 December 2018, Montr\u00e9al, Canada.7944\u20137954. http:\/\/papers.nips.cc\/paper\/8018-learning-to-repair-software-vulnerabilities-with-generative-adversarial-networks"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2012.6227135"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Thong Hoang Hong\u00a0Jin Kang David Lo and Julia Lawall. 2020. CC2Vec: Distributed Representations of Code Changes. In ICSE.","DOI":"10.1145\/3377811.3380361"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3196321.3196334"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1192"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380342"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00026"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3318162"},{"key":"e_1_3_2_1_25_1","volume-title":"On the Naturalness of Hardware Descriptions. In Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering. 530\u2013542","author":"Lee Jaeseong","year":"2020","unstructured":"Jaeseong Lee, Pengyu Nie, Junyi\u00a0Jessy Li, and Milos Gligoric. 2020. On the Naturalness of Hardware Descriptions. In Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering. 530\u2013542."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.5555\/3304222.3304348"},{"key":"e_1_3_2_1_27_1","volume-title":"Bridging semantic gaps between natural languages and APIs with word embedding","author":"Li Xiaochen","year":"2018","unstructured":"Xiaochen Li, He Jiang, Yasutaka Kamei, and Xin Chen. 2018. Bridging semantic gaps between natural languages and APIs with word embedding. IEEE Transactions on Software Engineering(2018)."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073445.1073465"},{"key":"e_1_3_2_1_29_1","unstructured":"Hui Liu Mingzhu Shen Jiaqi Zhu Nan Niu Ge Li and Lu Zhang. 2020. Deep Learning Based Program Generation from Requirements Text: Are We There Yet?IEEE Transactions on Software Engineering(2020)."},{"key":"e_1_3_2_1_30_1","volume-title":"TreeGAN: Syntax-Aware Sequence Generation with Generative Adversarial Networks. ArXiv e-prints","author":"Liu Xinyue","year":"2018","unstructured":"Xinyue Liu, Xiangnan Kong, Lei Liu, and Kuorong Chiang. 2018. TreeGAN: Syntax-Aware Sequence Generation with Generative Adversarial Networks. ArXiv e-prints (2018). arXiv:1808.07582"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3340455"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"crossref","unstructured":"Anh\u00a0Tuan Nguyen Trong\u00a0Duc Nguyen Hung\u00a0Dang Phan and Tien\u00a0N. Nguyen. 2018. A Deep Neural Network Language Model with Contexts for Source Code. In SANER.","DOI":"10.1109\/SANER.2018.8330220"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2015.74"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3540250.3549128"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2015.36"},{"key":"e_1_3_2_1_36_1","volume-title":"Annual Meeting on Association for Computational Linguistics (ACL). 311\u2013318","author":"Papineni Kishore","year":"2002","unstructured":"Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. BLEU: a method for automatic evaluation of machine translation. In Annual Meeting on Association for Computational Linguistics (ACL). 311\u2013318."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468623"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W18-6319"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3460348"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1162\/coli_a_00322"},{"key":"e_1_3_2_1_41_1","unstructured":"Shuo Ren Daya Guo Shuai Lu Long Zhou Shujie Liu Duyu Tang Ming Zhou Ambrosio Blanco and Shuai Ma. 2020. CodeBLEU: a Method for Automatic Evaluation of Code Synthesis. arXiv preprint arXiv:2009.10297(2020)."},{"key":"e_1_3_2_1_42_1","volume-title":"Proceedings of the ACL workshop on intrinsic and extrinsic evaluation measures for machine translation and\/or summarization. 57\u201364","author":"Riezler Stefan","year":"2005","unstructured":"Stefan Riezler and John\u00a0T Maxwell\u00a0III. 2005. On some pitfalls in automatic evaluation and significance testing for MT. In Proceedings of the ACL workshop on intrinsic and extrinsic evaluation measures for machine translation and\/or summarization. 57\u201364."},{"key":"e_1_3_2_1_43_1","volume-title":"Unsupervised Translation of Programming Languages. In Advances in Neural Information Processing Systems 33: Annual Conference on Neural Information Processing Systems 2020","author":"Rozi\u00e8re Baptiste","year":"2020","unstructured":"Baptiste Rozi\u00e8re, Marie-Anne Lachaux, Lowik Chanussot, and Guillaume Lample. 2020. Unsupervised Translation of Programming Languages. In Advances in Neural Information Processing Systems 33: Annual Conference on Neural Information Processing Systems 2020, NeurIPS 2020, December 6-12, 2020, virtual, Hugo Larochelle, Marc\u2019Aurelio Ranzato, Raia Hadsell, Maria-Florina Balcan, and Hsuan-Tien Lin (Eds.). https:\/\/proceedings.neurips.cc\/paper\/2020\/hash\/ed23fbf18c2cd35f8c7f8de44f85c08d-Abstract.html"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33017055"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME.2014.77"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3417058"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1109\/APSEC.2018.00020"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICPC.2019.00034"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00021"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238206"},{"key":"e_1_3_2_1_51_1","volume-title":"Detecting Code Clones with Graph Neural Network and Flow-Augmented Abstract Syntax Tree. In 2020 IEEE 27th International Conference on Software Analysis, Evolution and Reengineering (SANER)","author":"Wang Wenhan","unstructured":"Wenhan Wang, Ge Li, Bo Ma, Xin Xia, and Zhi Jin. 2020. Detecting Code Clones with Graph Neural Network and Flow-Augmented Abstract Syntax Tree. In 2020 IEEE 27th International Conference on Software Analysis, Evolution and Reengineering (SANER). IEEE, 261\u2013271."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"crossref","unstructured":"Cody Watson Michele Tufano Kevin Moran Gabriele Bavota and Denys Poshyvanyk. 2020. On Learning Meaningful Assert Statements for Unit Test Cases. In ICSE.","DOI":"10.1145\/3377811.3380429"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"crossref","unstructured":"Jian Zhang Xu Wang Hongyu Zhang Hailong Sun and Xudong Liu. 2020. Retrieval-based Neural Source Code Summarization. In ICSE.","DOI":"10.1145\/3377811.3380383"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416568"}],"event":{"name":"ASE '22: 37th IEEE\/ACM International Conference on Automated Software Engineering","location":"Rochester MI USA","acronym":"ASE '22"},"container-title":["Proceedings of the 37th IEEE\/ACM International Conference on Automated Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3551349.3556903","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3551349.3556903","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T08:26:21Z","timestamp":1755851181000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3551349.3556903"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,10,10]]},"references-count":54,"alternative-id":["10.1145\/3551349.3556903","10.1145\/3551349"],"URL":"https:\/\/doi.org\/10.1145\/3551349.3556903","relation":{},"subject":[],"published":{"date-parts":[[2022,10,10]]},"assertion":[{"value":"2023-01-05","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}