{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,17]],"date-time":"2026-04-17T23:08:38Z","timestamp":1776467318646,"version":"3.51.2"},"reference-count":42,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"7","license":[{"start":{"date-parts":[[2023,7,1]],"date-time":"2023-07-01T00:00:00Z","timestamp":1688169600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"},{"start":{"date-parts":[[2023,7,1]],"date-time":"2023-07-01T00:00:00Z","timestamp":1688169600000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/501100008982","name":"National Science Foundation","doi-asserted-by":"publisher","award":["CCF-2052696"],"award-info":[{"award-number":["CCF-2052696"]}],"id":[{"id":"10.13039\/501100008982","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IIEEE Trans. Software Eng."],"published-print":{"date-parts":[[2023,7]]},"DOI":"10.1109\/tse.2023.3267446","type":"journal-article","created":{"date-parts":[[2023,4,17]],"date-time":"2023-04-17T17:56:49Z","timestamp":1681754209000},"page":"3675-3691","source":"Crossref","is-referenced-by-count":85,"title":["MultiPL-E: A Scalable and Polyglot Approach to Benchmarking Neural Code Generation"],"prefix":"10.1109","volume":"49","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9318-7454","authenticated-orcid":false,"given":"Federico","family":"Cassano","sequence":"first","affiliation":[{"name":"Computer Science, Northeastern University, Boston, MA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0494-7245","authenticated-orcid":false,"given":"John","family":"Gouwar","sequence":"additional","affiliation":[{"name":"Computer Science, Northeastern University, Boston, MA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4609-0844","authenticated-orcid":false,"given":"Daniel","family":"Nguyen","sequence":"additional","affiliation":[{"name":"Hanover High School, Hanover, NH, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7053-699X","authenticated-orcid":false,"given":"Sydney","family":"Nguyen","sequence":"additional","affiliation":[{"name":"Computer Science, Wellesley College, Wellesley, MA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-4398-6987","authenticated-orcid":false,"given":"Luna","family":"Phipps-Costin","sequence":"additional","affiliation":[{"name":"Computer Science, Northeastern University, Boston, MA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8612-5178","authenticated-orcid":false,"given":"Donald","family":"Pinckney","sequence":"additional","affiliation":[{"name":"Computer Science, Northeastern University, Boston, MA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8008-8481","authenticated-orcid":false,"given":"Ming-Ho","family":"Yee","sequence":"additional","affiliation":[{"name":"Computer Science, Northeastern University, Boston, MA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2606-3280","authenticated-orcid":false,"given":"Yangtian","family":"Zi","sequence":"additional","affiliation":[{"name":"Computer Science, Northeastern University, Boston, MA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5717-4210","authenticated-orcid":false,"given":"Carolyn Jane","family":"Anderson","sequence":"additional","affiliation":[{"name":"Computer Science, Wellesley College, Wellesley, MA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5222-7720","authenticated-orcid":false,"given":"Molly Q","family":"Feldman","sequence":"additional","affiliation":[{"name":"Computer Science, Oberlin College, Oberlin, OH, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7493-3271","authenticated-orcid":false,"given":"Arjun","family":"Guha","sequence":"additional","affiliation":[{"name":"Computer Science, Northeastern University, Boston, MA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0014-7670","authenticated-orcid":false,"given":"Michael","family":"Greenberg","sequence":"additional","affiliation":[{"name":"Computer Science, Stevens Institute of Technology, Hoboken, NJ, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4849-6776","authenticated-orcid":false,"given":"Abhinav","family":"Jangda","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, WA, USA"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Evaluating large language models trained on code","author":"Chen","year":"2021"},{"key":"ref2","article-title":"Program synthesis with large language models","author":"Austin","year":"2021"},{"key":"ref3","article-title":"CodeGen: An open large language model for code with multi-turn program synthesis","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Nijkamp"},{"key":"ref4","article-title":"InCoder: A generative model for code infilling and synthesis","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Fried"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.bigscience-1.9"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1145\/3520312.3534862"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1145\/3520312.3534864"},{"key":"ref8","first-page":"11906","article-title":"SPoC: Search-based pseudocode to code","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Kulal"},{"key":"ref9","article-title":"Measuring coding challenge competence with APPS","volume-title":"Proc. Conf. Neural Inf. Process. Syst. Datasets Benchmarks Track","author":"Hendrycks"},{"key":"ref10","first-page":"476","article-title":"Learning to mine aligned code and natural language pairs from Stack Overflow","volume-title":"Proc. Int. Conf. Mining Softw. Repositories","author":"Yin"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/FMCAD.2013.6679385"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1561\/2500000049"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1561\/2500000010"},{"key":"ref14","first-page":"1877","article-title":"Language models are few-shot learners","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Brown"},{"key":"ref15","article-title":"GPT-J-6B: A 6 Billion Parameter Autoregressive Language Model","author":"Wang","year":"2021"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.728"},{"key":"ref18","article-title":"Unit test case generation with transformers and focal context","author":"Tufano","year":"2020"},{"key":"ref19","article-title":"CodeXGLUE: A machine learning benchmark dataset for code understanding and generation","volume-title":"Proc. Conf. Neural Inf. Process. Syst. Datasets Benchmarks Track","author":"Lu"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510049"},{"key":"ref21","article-title":"LambdaNet: Probabilistic type inference using graph neural networks","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Wei"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3236051"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409715"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2123433119"},{"key":"ref25","article-title":"The curious case of neural text degeneration","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Holtzman"},{"key":"ref26","article-title":"CodeBLEU: A method for automatic evaluation of code synthesis","author":"Ren","year":"2020"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1425"},{"key":"ref28","article-title":"Seq2SQL: Generating structured queries from natural language using reinforcement learning","author":"Zhong","year":"2017"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref30","article-title":"CM3: A causal masked multimodal model of the Internet","author":"Aghajanyan","year":"2022"},{"key":"ref31","article-title":"The Pile: An 800gb dataset of diverse text for language modeling","author":"Gao","year":"2021"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.18637\/jss.v067.i01"},{"key":"ref33","article-title":"SantaCoder: Dont reach for the stars!","volume-title":"Proc. Deep Learn. Code Workshop","author":"Allal"},{"key":"ref34","article-title":"CodeT: Code generation with generated tests","author":"Chen","year":"2022"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-eacl.20"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/d18-1192"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1425"},{"key":"ref39","volume-title":"Natural Language Processing With Transformers","author":"Tunstall","year":"2022"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1126\/science.abq1158"},{"key":"ref41","article-title":"PaLM: Scaling language modeling with pathways","author":"Chowdhery","year":"2022"},{"key":"ref42","article-title":"Multi-lingual evaluation of code generation models","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Athiwaratkun"}],"container-title":["IEEE Transactions on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/ieeexplore.ieee.org\/ielam\/32\/10185148\/10103177-aam.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/32\/10185148\/10103177.pdf?arnumber=10103177","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,5,28]],"date-time":"2024-05-28T04:24:54Z","timestamp":1716870294000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10103177\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7]]},"references-count":42,"journal-issue":{"issue":"7"},"URL":"https:\/\/doi.org\/10.1109\/tse.2023.3267446","relation":{},"ISSN":["0098-5589","1939-3520","2326-3881"],"issn-type":[{"value":"0098-5589","type":"print"},{"value":"1939-3520","type":"electronic"},{"value":"2326-3881","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,7]]}}}