{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,21]],"date-time":"2026-01-21T08:23:39Z","timestamp":1768983819767,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":37,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,20]],"date-time":"2024-04-20T00:00:00Z","timestamp":1713571200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,20]]},"DOI":"10.1145\/3643795.3648395","type":"proceedings-article","created":{"date-parts":[[2024,9,10]],"date-time":"2024-09-10T13:46:19Z","timestamp":1725975979000},"page":"62-69","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["PromptSet: A Programmer's Prompting Dataset"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0004-3144-8235","authenticated-orcid":false,"given":"Kaiser","family":"Pister","sequence":"first","affiliation":[{"name":"University of Wisconsin-Madison, Madison, Wisconsin, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-0817-6898","authenticated-orcid":false,"given":"Dhruba Jyoti","family":"Paul","sequence":"additional","affiliation":[{"name":"University of Wisconsin-Madison, Madison, Wisconsin, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-1298-2279","authenticated-orcid":false,"given":"Ishan","family":"Joshi","sequence":"additional","affiliation":[{"name":"University of Wisconsin-Madison, Madison, Wisconsin, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-0049-6736","authenticated-orcid":false,"given":"Patrick","family":"Brophy","sequence":"additional","affiliation":[{"name":"University of Wisconsin-Madison, Madison, Wisconsin, USA"}]}],"member":"320","published-online":{"date-parts":[[2024,9,10]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"LangChain AI. 2023. LangServe. https:\/\/github.com\/langchain-ai\/langserve"},{"key":"e_1_3_2_1_2_1","unstructured":"Anthropic. 2023. Claude 2.1 Prompting. https:\/\/www.anthropic.com\/index\/claude-2-1-prompting"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"crossref","unstructured":"Maciej Besta Nils Blach Ales Kubicek Robert Gerstenberger Lukas Gianinazzi Joanna Gajda Tomasz Lehmann Michal Podstawski Hubert Niewiadomski Piotr Nyczyk and Torsten Hoefler. 2023. Graph of Thoughts: Solving Elaborate Problems with Large Language Models. arXiv:2308.09687 [cs.CL]","DOI":"10.1609\/aaai.v38i16.29720"},{"key":"e_1_3_2_1_4_1","volume-title":"Lin (Eds.)","volume":"33","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, Sandhini Agarwal, Ariel Herbert-Voss, Gretchen Krueger, Tom Henighan, Rewon Child, Aditya Ramesh, Daniel Ziegler, Jeffrey Wu, Clemens Winter, Chris Hesse, Mark Chen, Eric Sigler, Mateusz Litwin, Scott Gray, Benjamin Chess, Jack Clark, Christopher Berner, Sam McCandlish, Alec Radford, Ilya Sutskever, and Dario Amodei. 2020. Language Models are Few-Shot Learners. In Advances in Neural Information Processing Systems, H. Larochelle, M. Ranzato, R. Hadsell, M.F. Balcan, and H. Lin (Eds.), Vol. 33. Curran Associates, Inc., 1877--1901. https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2020\/file\/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf"},{"key":"e_1_3_2_1_5_1","unstructured":"Tom B. Brown Benjamin Mann Nick Ryder Melanie Subbiah Jared Kaplan Prafulla Dhariwal Arvind Neelakantan Pranav Shyam Girish Sastry Amanda Askell Sandhini Agarwal Ariel Herbert-Voss Gretchen Krueger Tom Henighan Rewon Child Aditya Ramesh Daniel M. Ziegler Jeffrey Wu Clemens Winter Christopher Hesse Mark Chen Eric Sigler Mateusz Litwin Scott Gray Benjamin Chess Jack Clark Christopher Berner Sam McCandlish Alec Radford Ilya Sutskever and Dario Amodei. 2020. Language Models are Few-Shot Learners. arXiv:2005.14165 [cs.CL]"},{"key":"e_1_3_2_1_6_1","unstructured":"Miguel Corralm. 2023. Awesome Prompting. https:\/\/github.com\/corralm\/awesome-prompting"},{"key":"e_1_3_2_1_7_1","unstructured":"CSpell. 2023. CSpell. https:\/\/www.npmjs.com\/package\/cspell"},{"key":"e_1_3_2_1_8_1","unstructured":"Python Software Foundation. 2023. Black. https:\/\/github.com\/psf\/black"},{"key":"e_1_3_2_1_9_1","volume-title":"Balancing Autonomy and Alignment: A Multi-Dimensional Taxonomy for Autonomous LLM-powered Multi-Agent Architectures. ArXiv abs\/2310.03659","author":"H\u00e4ndler Thorsten","year":"2023","unstructured":"Thorsten H\u00e4ndler. 2023. Balancing Autonomy and Alignment: A Multi-Dimensional Taxonomy for Autonomous LLM-powered Multi-Agent Architectures. ArXiv abs\/2310.03659 (2023). https:\/\/api.semanticscholar.org\/CorpusID:263671545"},{"key":"e_1_3_2_1_10_1","volume-title":"Generative Models as a Complex Systems Science: How can we make sense of large language model behavior? preprint","author":"Holtzman Ari","year":"2023","unstructured":"Ari Holtzman, Peter West, and Luke Zettlemoyer. 2023. Generative Models as a Complex Systems Science: How can we make sense of large language model behavior? preprint (2023)."},{"key":"e_1_3_2_1_11_1","volume-title":"LoRA: Low-Rank Adaptation of Large Language Models. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=nZeVKeeFYf9","author":"Hu Edward J","year":"2022","unstructured":"Edward J Hu, Yelong Shen, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, and Weizhu Chen. 2022. LoRA: Low-Rank Adaptation of Large Language Models. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=nZeVKeeFYf9"},{"key":"e_1_3_2_1_12_1","volume-title":"Catastrophic Jailbreak of Open-source LLMs via Exploiting Generation. arXiv preprint arXiv:2310.06987","author":"Huang Yangsibo","year":"2023","unstructured":"Yangsibo Huang, Samyak Gupta, Mengzhou Xia, Kai Li, and Danqi Chen. 2023. Catastrophic Jailbreak of Open-source LLMs via Exploiting Generation. arXiv preprint arXiv:2310.06987 (2023)."},{"key":"e_1_3_2_1_13_1","unstructured":"Instagram. 2016. Continuous Deployment at Instagram. https:\/\/instagram-engineering.com\/continuous-deployment-at-instagram-1e18548f01d1"},{"key":"e_1_3_2_1_14_1","unstructured":"Albert Q. Jiang Alexandre Sablayrolles Arthur Mensch Chris Bamford Devendra Singh Chaplot Diego de las Casas Florian Bressand Gianna Lengyel Guillaume Lample Lucile Saulnier L\u00e9lio Renard Lavaud Marie-Anne Lachaux Pierre Stock Teven Le Scao Thibaut Lavril Thomas Wang Timoth\u00e9e Lacroix and William El Sayed. 2023. Mistral 7B. arXiv:2310.06825 [cs.CL]"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"Zhengbao Jiang Frank F. Xu Jun Araki and Graham Neubig. 2020. How Can We Know What Language Models Know? arXiv:1911.12543 [cs.CL]","DOI":"10.1162\/tacl_a_00324"},{"key":"e_1_3_2_1_16_1","volume-title":"Compressing text classification models. arXiv preprint arXiv:1612.03651","author":"Joulin Armand","year":"2016","unstructured":"Armand Joulin, Edouard Grave, Piotr Bojanowski, Matthijs Douze, H\u00e9rve J\u00e9gou, and Tomas Mikolov. 2016. FastText.zip: Compressing text classification models. arXiv preprint arXiv:1612.03651 (2016)."},{"key":"e_1_3_2_1_17_1","volume-title":"Bag of Tricks for Efficient Text Classification. arXiv preprint arXiv:1607.01759","author":"Joulin Armand","year":"2016","unstructured":"Armand Joulin, Edouard Grave, Piotr Bojanowski, and Tomas Mikolov. 2016. Bag of Tricks for Efficient Text Classification. arXiv preprint arXiv:1607.01759 (2016)."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"crossref","unstructured":"Yao Lu Max Bartolo Alastair Moore Sebastian Riedel and Pontus Stenetorp. 2022. Fantastically Ordered Prompts and Where to Find Them: Overcoming Few-Shot Prompt Order Sensitivity. arXiv:2104.08786 [cs.CL]","DOI":"10.18653\/v1\/2022.acl-long.556"},{"key":"e_1_3_2_1_19_1","unstructured":"Rajasekhar Reddy Mekala Yasaman Razeghi and Sameer Singh. 2023. EchoPrompt: Instructing the Model to Rephrase Queries for Improved In-context Learning. arXiv:2309.10687 [cs.CL]"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"Sewon Min Xinxi Lyu Ari Holtzman Mikel Artetxe Mike Lewis Hannaneh Hajishirzi and Luke Zettlemoyer. 2022. Rethinking the Role of Demonstrations: What Makes In-Context Learning Work? arXiv:2202.12837 [cs.CL]","DOI":"10.18653\/v1\/2022.emnlp-main.759"},{"key":"e_1_3_2_1_22_1","unstructured":"Mary Phuong and Marcus Hutter. 2022. Formal Algorithms for Transformers. arXiv:2207.09238 [cs.LG]"},{"key":"e_1_3_2_1_23_1","volume-title":"Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks. CoRR abs\/1908.10084","author":"Reimers Nils","year":"2019","unstructured":"Nils Reimers and Iryna Gurevych. 2019. Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks. CoRR abs\/1908.10084 (2019). arXiv:1908.10084 http:\/\/arxiv.org\/abs\/1908.10084"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"crossref","unstructured":"Laria Reynolds and Kyle McDonell. 2021. Prompt Programming for Large Language Models: Beyond the Few-Shot Paradigm. arXiv:2102.07350 [cs.CL]","DOI":"10.1145\/3411763.3451760"},{"key":"e_1_3_2_1_25_1","unstructured":"SquidgyAI. 2023. Squidgy Testy. https:\/\/github.com\/squidgyai\/squidgy-testy"},{"key":"e_1_3_2_1_26_1","volume-title":"Beyond Memorization: Violating Privacy Via Inference with Large Language Models. arXiv:2310.07298 [cs.AI]","author":"Staab Robin","year":"2023","unstructured":"Robin Staab, Mark Vero, Mislav Balunovi\u0107, and Martin Vechev. 2023. Beyond Memorization: Violating Privacy Via Inference with Large Language Models. arXiv:2310.07298 [cs.AI]"},{"key":"e_1_3_2_1_27_1","volume-title":"Hashimoto","author":"Taori Rohan","year":"2023","unstructured":"Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li, Carlos Guestrin, Percy Liang, and Tatsunori B. Hashimoto. 2023. Stanford Alpaca: An Instruction-following LLaMA model. https:\/\/github.com\/tatsu-lab\/stanford_alpaca."},{"key":"e_1_3_2_1_28_1","unstructured":"Hugo Touvron Louis Martin Kevin Stone Peter Albert Amjad Almahairi Yasmine Babaei Nikolay Bashlykov Soumya Batra Prajjwal Bhargava Shruti Bhosale Dan Bikel Lukas Blecher Cristian Canton Ferrer Moya Chen Guillem Cucurull David Esiobu Jude Fernandes Jeremy Fu Wenyin Fu Brian Fuller Cynthia Gao Vedanuj Goswami Naman Goyal Anthony Hartshorn Saghar Hosseini Rui Hou Hakan Inan Marcin Kardas Viktor Kerkez Madian Khabsa Isabel Kloumann Artem Korenev Punit Singh Koura Marie-Anne Lachaux Thibaut Lavril Jenya Lee Diana Liskovich Yinghai Lu Yuning Mao Xavier Martinet Todor Mihaylov Pushkar Mishra Igor Molybog Yixin Nie Andrew Poulton Jeremy Reizenstein Rashi Rungta Kalyan Saladi Alan Schelten Ruan Silva Eric Michael Smith Ranjan Subramanian Xiaoqing Ellen Tan Binh Tang Ross Taylor Adina Williams Jian Xiang Kuan Puxin Xu Zheng Yan Iliyan Zarov Yuchen Zhang Angela Fan Melanie Kambadur Sharan Narang Aurelien Rodriguez Robert Stojnic Sergey Edunov and Thomas Scialom. 2023. Llama 2: Open Foundation and Fine-Tuned Chat Models. arXiv:2307.09288 [cs.CL]"},{"key":"e_1_3_2_1_29_1","unstructured":"Traceloop. 2023. OpenTelemetry. https:\/\/www.traceloop.com\/blog\/diy-observability-for-llm-with-opentelemetry"},{"key":"e_1_3_2_1_30_1","unstructured":"tree sitter. [n. d.]. Tree-sitter. https:\/\/tree-sitter.github.io\/tree-sitter"},{"key":"e_1_3_2_1_31_1","volume-title":"\u0141 ukasz Kaiser, and Illia Polosukhin","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, \u0141 ukasz Kaiser, and Illia Polosukhin. 2017. Attention is All you Need. In Advances in Neural Information Processing Systems, I. Guyon, U. Von Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett (Eds.), Vol. 30. Curran Associates, Inc. https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2017\/file\/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf"},{"key":"e_1_3_2_1_32_1","volume-title":"Chi, Quoc Le, and Denny Zhou","author":"Wei Jason","year":"2023","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Brian Ichter, Fei Xia, Ed Chi, Quoc Le, and Denny Zhou. 2023. Chain-of-Thought Prompting Elicits Reasoning in Large Language Models. arXiv:2201.11903 [cs.CL]"},{"key":"e_1_3_2_1_33_1","volume-title":"Schmidt","author":"White Jules","year":"2023","unstructured":"Jules White, Quchen Fu, Sam Hays, Michael Sandborn, Carlos Olea, Henry Gilbert, Ashraf Elnashar, Jesse Spencer-Smith, and Douglas C. Schmidt. 2023. A Prompt Pattern Catalog to Enhance Prompt Engineering with ChatGPT. arXiv:2302.11382 [cs.SE]"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3643991.3648400"},{"key":"e_1_3_2_1_35_1","unstructured":"Chengrun Yang Xuezhi Wang Yifeng Lu Hanxiao Liu Quoc V. Le Denny Zhou and Xinyun Chen. 2023. Large Language Models as Optimizers. arXiv:2309.03409 [cs.LG]"},{"key":"e_1_3_2_1_36_1","unstructured":"Seonghyeon Ye Hyeonbin Hwang Sohee Yang Hyeongu Yun Yireun Kim and Minjoon Seo. 2023. In-Context Instruction Learning. arXiv:arXiv:2302.14691"},{"key":"e_1_3_2_1_37_1","unstructured":"Tony Z. Zhao Eric Wallace Shi Feng Dan Klein and Sameer Singh. 2021. Calibrate Before Use: Improving Few-Shot Performance of Language Models. arXiv:2102.09690 [cs.CL]"},{"key":"e_1_3_2_1_38_1","volume-title":"Ziwen Han, Keiran Paster, Silviu Pitis, Harris Chan, and Jimmy Ba.","author":"Zhou Yongchao","year":"2022","unstructured":"Yongchao Zhou, Andrei Ioan Muresanu, Ziwen Han, Keiran Paster, Silviu Pitis, Harris Chan, and Jimmy Ba. 2022. Large Language Models Are Human-Level Prompt Engineers. (2022). arXiv:2211.01910 [cs.LG]"}],"event":{"name":"LLM4Code '24: 1st International Workshop on Large Language Models for Code","location":"Lisbon Portugal","acronym":"LLM4Code '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"]},"container-title":["Proceedings of the 1st International Workshop on Large Language Models for Code"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3643795.3648395","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3643795.3648395","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:57:45Z","timestamp":1750294665000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3643795.3648395"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,20]]},"references-count":37,"alternative-id":["10.1145\/3643795.3648395","10.1145\/3643795"],"URL":"https:\/\/doi.org\/10.1145\/3643795.3648395","relation":{},"subject":[],"published":{"date-parts":[[2024,4,20]]},"assertion":[{"value":"2024-09-10","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}