{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,9]],"date-time":"2026-06-09T08:43:12Z","timestamp":1780994592754,"version":"3.54.1"},"publisher-location":"New York, NY, USA","reference-count":46,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,5,21]],"date-time":"2022-05-21T00:00:00Z","timestamp":1653091200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,5,21]]},"DOI":"10.1145\/3510003.3510203","type":"proceedings-article","created":{"date-parts":[[2022,7,5]],"date-time":"2022-07-05T22:42:59Z","timestamp":1657060979000},"page":"1219-1231","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":135,"title":["Jigsaw"],"prefix":"10.1145","author":[{"given":"Naman","family":"Jain","sequence":"first","affiliation":[{"name":"Microsoft Research, Bangalore, India"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Skanda","family":"Vaidyanath","sequence":"additional","affiliation":[{"name":"Stanford University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Arun","family":"Iyer","sequence":"additional","affiliation":[{"name":"Microsoft Research, Bangalore, India"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nagarajan","family":"Natarajan","sequence":"additional","affiliation":[{"name":"Microsoft Research, Bangalore, India"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Suresh","family":"Parthasarathy","sequence":"additional","affiliation":[{"name":"Microsoft Research, Bangalore, India"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sriram","family":"Rajamani","sequence":"additional","affiliation":[{"name":"Microsoft Research, Bangalore, India"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rahul","family":"Sharma","sequence":"additional","affiliation":[{"name":"Microsoft Research, Bangalore, India"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2022,7,5]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"[n. d.]. GitHub Copilot \u00b7 Your AI pair programmer. https:\/\/copilot.github.com\/"},{"key":"e_1_3_2_1_2_1","unstructured":"[n. d.]. Jupyter. https:\/\/jupyter.org\/"},{"key":"e_1_3_2_1_3_1","unstructured":"[n. d.]. Parenthesis Blog. https:\/\/www.roelpeters.be\/cannot-compare-a-dtyped-object-array-with-a-scalar-of-type-bool\/"},{"key":"e_1_3_2_1_4_1","unstructured":"[n. d.]. Parenthesis StackOverflow. https:\/\/stackoverflow.com\/questions\/38252423\/python-error-typeerror-cannot-compare-a-dtyped-float64-array-with-a-scalar-o"},{"key":"e_1_3_2_1_5_1","unstructured":"[n. d.]. Spider 1.0: Yale Semantic Parsing and Text-to-SQL Challenge."},{"key":"e_1_3_2_1_6_1","unstructured":"[n. d.]. TensorFlow. https:\/\/www.tensorflow.org\/"},{"key":"e_1_3_2_1_7_1","volume-title":"Program Synthesis with Large Language Models. ArXiv abs\/2108.07732","author":"Austin Jacob","year":"2021","unstructured":"Jacob Austin, Augustus Odena, Maxwell Nye, Maarten Bosma, Henryk Michalewski, David Dohan, Ellen Jiang, Carrie Cai, Michael Terry, Quoc Le, and Charles Sutton. 2021. Program Synthesis with Large Language Models. ArXiv abs\/2108.07732 (2021)."},{"key":"e_1_3_2_1_8_1","volume-title":"5th International Conference on Learning Representations, ICLR 2017, Toulon, France, April 24--26, 2017, Conference Track Proceedings. OpenReview.net.","author":"Balog Matej","year":"2017","unstructured":"Matej Balog, Alexander L. Gaunt, Marc Brockschmidt, Sebastian Nowozin, and Daniel Tarlow. 2017. DeepCoder: Learning to Write Programs. In 5th International Conference on Learning Representations, ICLR 2017, Toulon, France, April 24--26, 2017, Conference Track Proceedings. OpenReview.net."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3360594"},{"key":"e_1_3_2_1_10_1","volume-title":"Advances in Neural Information Processing Systems 33: Annual Conference on Neural Information Processing Systems 2020","author":"Brown Tom B.","year":"2020","unstructured":"Tom B. Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, Sandhini Agarwal, Ariel Herbert-Voss, Gretchen Krueger, Tom Henighan, Rewon Child, Aditya Ramesh, Daniel M. Ziegler, Jeffrey Wu, Clemens Winter, Christopher Hesse, Mark Chen, Eric Sigler, Mateusz Litwin, Scott Gray, Benjamin Chess, Jack Clark, Christopher Berner, Sam McCandlish, Alec Radford, Ilya Sutskever, and Dario Amodei. 2020. Language Models are Few-Shot Learners. In Advances in Neural Information Processing Systems 33: Annual Conference on Neural Information Processing Systems 2020, NeurIPS 2020, December 6--12, 2020, virtual, Hugo Larochelle, Marc'Aurelio Ranzato, Raia Hadsell, Maria-Florina Balcan, and Hsuan-Tien Lin (Eds.)."},{"key":"e_1_3_2_1_11_1","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique Ponde de Oliveira Pinto Jared Kaplan Harrison Edwards Yuri Burda Nicholas Joseph Greg Brockman Alex Ray Raul Puri Gretchen Krueger Michael Petrov Heidy Khlaaf Girish Sastry Pamela Mishkin Brooke Chan Scott Gray Nick Ryder Mikhail Pavlov Alethea Power Lukasz Kaiser Mohammad Bavarian Clemens Winter Philippe Tillet Felipe Petroski Such Dave Cummings Matthias Plappert Fotios Chantzis Elizabeth Barnes Ariel Herbert-Voss William Hebgen Guss Alex Nichol Alex Paino Nikolas Tezak Jie Tang Igor Babuschkin Suchir Balaji Shantanu Jain William Saunders Christopher Hesse Andrew N. Carr Jan Leike Joshua Achiam Vedant Misra Evan Morikawa Alec Radford Matthew Knight Miles Brundage Mira Murati Katie Mayer Peter Welinder Bob McGrew Dario Amodei Sam McCandlish Ilya Sutskever and Wojciech Zaremba. 2021. Evaluating Large Language Models Trained on Code. CoRR abs\/2107.03374 (2021)."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3453483.3454047"},{"key":"e_1_3_2_1_13_1","volume-title":"Proceedings of the 41st ACM SIGPLAN International Conference on Programming Language Design and Implementation, PLDI 2020","author":"Chen Qiaochu","year":"2020","unstructured":"Qiaochu Chen, Xinyu Wang, Xi Ye, Greg Durrett, and Isil Dillig. 2020. Multimodal synthesis of regular expressions. In Proceedings of the 41st ACM SIGPLAN International Conference on Programming Language Design and Implementation, PLDI 2020, London, UK, June 15--20, 2020, Alastair F. Donaldson and Emina Torlak (Eds.). ACM, 487--502."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338951"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.728"},{"key":"e_1_3_2_1_16_1","volume-title":"Learning Syntactic Program Transformations from Examples. CoRR abs\/1608.09000","author":"de Sousa Reudismam Rolim","year":"2016","unstructured":"Reudismam Rolim de Sousa, Gustavo Soares, Loris D'Antoni, Oleksandr Polozov, Sumit Gulwani, Rohit Gheyi, Ryo Suzuki, and Bjoern Hartmann. 2016. Learning Syntactic Program Transformations from Examples. CoRR abs\/1608.09000 (2016)."},{"key":"e_1_3_2_1_17_1","volume-title":"Proceedings of the 34th International Conference on Machine Learning, ICML 2017","author":"Devlin Jacob","year":"2017","unstructured":"Jacob Devlin, Jonathan Uesato, Surya Bhupatiraju, Rishabh Singh, Abdel-rahman Mohamed, and Pushmeet Kohli. 2017. RobustFill: Neural Program Learning under Noisy I\/O. In Proceedings of the 34th International Conference on Machine Learning, ICML 2017, Sydney, NSW, Australia, 6--11 August 2017 (Proceedings of Machine Learning Research, Vol. 70), Doina Precup and Yee Whye Teh (Eds.). PMLR, 990--998."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3192366.3192382"},{"key":"e_1_3_2_1_19_1","first-page":"3","article-title":"Programming by examples","volume":"45","author":"Gulwani Sumit","year":"2016","unstructured":"Sumit Gulwani. 2016. Programming by examples. Dependable Software Systems Engineering 45, 137 (2016), 3--15.","journal-title":"Dependable Software Systems Engineering"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.14778\/3231751.3231766"},{"key":"e_1_3_2_1_21_1","volume-title":"6th International Conference on Learning Representations, ICLR 2018, Vancouver, BC, Canada, April 30 - May 3, 2018, Conference Track Proceedings. OpenReview.net.","author":"Kalyan Ashwin","year":"2018","unstructured":"Ashwin Kalyan, Abhishek Mohta, Oleksandr Polozov, Dhruv Batra, Prateek Jain, and Sumit Gulwani. 2018. Neural-Guided Deductive Search for Real-Time Program Synthesis from Examples. In 6th International Conference on Learning Representations, ICLR 2018, Vancouver, BC, Canada, April 30 - May 3, 2018, Conference Track Proceedings. OpenReview.net."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3192366.3192410"},{"key":"e_1_3_2_1_23_1","volume-title":"prompt, and predict: A systematic survey of prompting methods in natural language processing. arXiv preprint arXiv:2107.13586","author":"Liu Pengfei","year":"2021","unstructured":"Pengfei Liu, Weizhe Yuan, Jinlan Fu, Zhengbao Jiang, Hiroaki Hayashi, and Graham Neubig. 2021. Pre-train, prompt, and predict: A systematic survey of prompting methods in natural language processing. arXiv preprint arXiv:2107.13586 (2021)."},{"key":"e_1_3_2_1_24_1","volume-title":"Proceedings of the Twenty-Seventh AAAI Conference on Artificial Intelligence, July 14--18","author":"Manshadi Mehdi Hafezi","year":"2013","unstructured":"Mehdi Hafezi Manshadi, Daniel Gildea, and James F. Allen. 2013. Integrating Programming by Example and Natural Language Programming. In Proceedings of the Twenty-Seventh AAAI Conference on Artificial Intelligence, July 14--18, 2013, Bellevue, Washington, USA, Marie desJardins and Michael L. Littman (Eds.). AAAI Press."},{"key":"e_1_3_2_1_25_1","volume-title":"Proceedings of the 30th International Conference on Machine Learning, ICML 2013, Atlanta, GA, USA, 16--21 June 2013 (JMLR Workshop and Conference Proceedings","volume":"195","author":"Menon Aditya Krishna","year":"2013","unstructured":"Aditya Krishna Menon, Omer Tamuz, Sumit Gulwani, Butler W. Lampson, and Adam Kalai. 2013. A Machine Learning Framework for Programming by Example. In Proceedings of the 30th International Conference on Machine Learning, ICML 2013, Atlanta, GA, USA, 16--21 June 2013 (JMLR Workshop and Conference Proceedings, Vol. 28). JMLR.org, 187--195."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"crossref","unstructured":"Anders Miltner Sumit Gulwani Vu Le Alan Leung Arjun Radhakrishna Gustavo Soares Ashish Tiwari and Abhishek Udupa. 2019. On the fly synthesis of edit suggestions. In Object-Oriented Programming Systems Languages & Applications (OOPSLA). ACM. https:\/\/www.microsoft.com\/en-us\/research\/publication\/on-the-fly-synthesis-of-edit-suggestions\/","DOI":"10.1145\/3360569"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","unstructured":"The pandas development team. 2020. pandas-dev\/pandas: Pandas. 10.5281\/zenodo.3509134","DOI":"10.5281\/zenodo.3509134"},{"key":"e_1_3_2_1_28_1","volume-title":"Neuro-Symbolic Program Synthesis. In 5th International Conference on Learning Representations, ICLR 2017, Toulon, France, April 24--26, 2017, Conference Track Proceedings. OpenReview.net.","author":"Parisotto Emilio","year":"2017","unstructured":"Emilio Parisotto, Abdel-rahman Mohamed, Rishabh Singh, Lihong Li, Dengyong Zhou, and Pushmeet Kohli. 2017. Neuro-Symbolic Program Synthesis. In 5th International Conference on Learning Representations, ICLR 2017, Toulon, France, April 24--26, 2017, Conference Track Proceedings. OpenReview.net."},{"key":"e_1_3_2_1_29_1","unstructured":"H. Pearce Baleegh Ahmad Benjamin Tan Brendan Dolan-Gavitt and R. Karri. 2021. An Empirical Cybersecurity Evaluation of GitHub Copilot's Code Contributions. ArXiv abs\/2108.09293 (2021)."},{"key":"e_1_3_2_1_30_1","volume-title":"True Few-Shot Learning with Language Models. ArXiv abs\/2105.11447","author":"Perez Ethan","year":"2021","unstructured":"Ethan Perez, Douwe Kiela, and Kyunghyun Cho. 2021. True Few-Shot Learning with Language Models. ArXiv abs\/2105.11447 (2021)."},{"key":"e_1_3_2_1_31_1","volume-title":"International Conference on Learning Representations.","author":"Poesia Gabriel","year":"2022","unstructured":"Gabriel Poesia, Alex Polozov, Vu Le, Ashish Tiwari, Gustavo Soares, Christopher Meek, and Sumit Gulwani. 2022. Synchromesh: Reliable Code Generation from Pre-trained Language Models. In International Conference on Learning Representations."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2814270.2814310"},{"key":"e_1_3_2_1_33_1","unstructured":"Alec Radford Jeff Wu Rewon Child David Luan Dario Amodei and Ilya Sutskever. 2019. Language Models are Unsupervised Multitask Learners. (2019)."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"crossref","unstructured":"Kia Rahmani Mohammad Raza Sumit Gulwani Vu Le Dan Morris Arjun Radhakrishna Gustavo Soares and Ashish Tiwari. 2021. Multi-modal Program Inference: a Marriage of Pre-trained Language Models and Component-based Synthesis. In OOPSLA.","DOI":"10.1145\/3485535"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/2594291.2594321"},{"key":"e_1_3_2_1_36_1","volume-title":"Proceedings of the Twenty-Fourth International Joint Conference on Artificial Intelligence, IJCAI 2015","author":"Raza Mohammad","year":"2015","unstructured":"Mohammad Raza, Sumit Gulwani, and Natasa Milic-Frayling. 2015. Compositional Program Synthesis from Natural Language and Examples. In Proceedings of the Twenty-Fourth International Joint Conference on Artificial Intelligence, IJCAI 2015, Buenos Aires, Argentina, July 25--31, 2015, Qiang Yang and Michael J. Wooldridge (Eds.). AAAI Press, 792--800."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1410"},{"key":"e_1_3_2_1_38_1","volume-title":"Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2021","author":"Rubin Ohad","year":"2021","unstructured":"Ohad Rubin and Jonathan Berant. 2021. SmBoP: Semi-autoregressive Bottom-up Semantic Parsing. In Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2021, Online, June 6--11, 2021, Kristina Toutanova, Anna Rumshisky, Luke Zettlemoyer, Dilek Hakkani-T\u00fcr, Iz Beltagy, Steven Bethard, Ryan Cotterell, Tanmoy Chakraborty, and Yichao Zhou (Eds.). Association for Computational Linguistics, 311--324."},{"key":"e_1_3_2_1_39_1","unstructured":"Gerard Salton and Michael J McGill. 1986. Introduction to modern information retrieval. (1986)."},{"key":"e_1_3_2_1_40_1","volume-title":"TF-Coder: Program Synthesis for Tensor Manipulations. CoRR abs\/2003.09040","author":"Shi Kensen","year":"2020","unstructured":"Kensen Shi, David Bieber, and Rishabh Singh. 2020. TF-Coder: Program Synthesis for Tensor Manipulations. CoRR abs\/2003.09040 (2020)."},{"key":"e_1_3_2_1_41_1","volume-title":"CAV 2015, San Francisco, CA, USA, July 18--24, 2015, Proceedings, Part I (Lecture Notes in Computer Science","volume":"414","author":"Singh Rishabh","year":"2015","unstructured":"Rishabh Singh and Sumit Gulwani. 2015. Predicting a Correct Program in Programming by Example. In Computer Aided Verification - 27th International Conference, CAV 2015, San Francisco, CA, USA, July 18--24, 2015, Proceedings, Part I (Lecture Notes in Computer Science, Vol. 9206), Daniel Kroening and Corina S. Pasareanu (Eds.). Springer, 398--414."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.677"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3379337.3415851"},{"key":"e_1_3_2_1_44_1","volume-title":"Sketch-Driven Regular Expression Generation from Natural Language and Examples. CoRR abs\/1908.05848","author":"Ye Xi","year":"2019","unstructured":"Xi Ye, Qiaochu Chen, Xinyu Wang, Isil Dillig, and Greg Durrett. 2019. Sketch-Driven Regular Expression Generation from Natural Language and Examples. CoRR abs\/1908.05848 (2019)."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1425"},{"key":"e_1_3_2_1_46_1","volume-title":"Calibrate Before Use: Improving Few-shot Performance of Language Models. In International Conference on Machine Learning.","author":"Zhao Tony Z.","year":"2021","unstructured":"Tony Z. Zhao, Eric Wallace, Shi Feng, Dan Klein, and Sameer Singh. 2021. Calibrate Before Use: Improving Few-shot Performance of Language Models. In International Conference on Machine Learning."}],"event":{"name":"ICSE '22: 44th International Conference on Software Engineering","location":"Pittsburgh Pennsylvania","acronym":"ICSE '22","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 44th International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3510003.3510203","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3510003.3510203","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:12:24Z","timestamp":1750191144000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3510003.3510203"}},"subtitle":["large language models meet program synthesis"],"short-title":[],"issued":{"date-parts":[[2022,5,21]]},"references-count":46,"alternative-id":["10.1145\/3510003.3510203","10.1145\/3510003"],"URL":"https:\/\/doi.org\/10.1145\/3510003.3510203","relation":{},"subject":[],"published":{"date-parts":[[2022,5,21]]},"assertion":[{"value":"2022-07-05","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}