{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,17]],"date-time":"2026-10-17T09:56:04Z","timestamp":1792230964028,"version":"4.4.0"},"publisher-location":"New York, NY, USA","reference-count":77,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,6,23]]},"DOI":"10.1145\/3696630.3728533","type":"proceedings-article","created":{"date-parts":[[2025,7,28]],"date-time":"2025-07-28T19:08:09Z","timestamp":1753729689000},"page":"75-86","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":31,"title":["From Prompts to Templates: A Systematic Prompt Template Analysis for Real-world LLMapps"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0000-8485-3577","authenticated-orcid":false,"given":"Yuetian","family":"Mao","sequence":"first","affiliation":[{"name":"Technical University of Munich, Heilbronn, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-1501-8009","authenticated-orcid":false,"given":"Junjie","family":"He","sequence":"additional","affiliation":[{"name":"Technical University of Munich, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2011-9618","authenticated-orcid":false,"given":"Chunyang","family":"Chen","sequence":"additional","affiliation":[{"name":"Technical University of Munich, Heilbronn, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,7,28]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Diogo Almeida, Janko Altenschmidt, Sam Altman, Shyamal Anadkat, et al.","author":"Achiam Josh","year":"2023","unstructured":"Josh Achiam, Steven Adler, Sandhini Agarwal, Lama Ahmad, Ilge Akkaya, Florencia Leoni Aleman, Diogo Almeida, Janko Altenschmidt, Sam Altman, Shyamal Anadkat, et al. 2023. Gpt-4 technical report. arXiv preprint arXiv:2303.08774 (2023)."},{"key":"e_1_3_2_1_2_1","volume-title":"SC22: International Conference for High Performance Computing, Networking, Storage and Analysis. IEEE, 1\u201315","author":"Aminabadi Reza Yazdani","year":"2022","unstructured":"Reza Yazdani Aminabadi, Samyam Rajbhandari, Ammar Ahmad Awan, Cheng Li, Du Li, Elton Zheng, Olatunji Ruwase, Shaden Smith, Minjia Zhang, Jeff Rasley, et al. 2022. Deepspeed-inference: enabling efficient inference of transformer models at unprecedented scale. In SC22: International Conference for High Performance Computing, Networking, Storage and Analysis. IEEE, 1\u201315."},{"key":"e_1_3_2_1_3_1","unstructured":"Anthropic. 2024. User Guides - Prompt engineering. https:\/\/docs.anthropic.com\/en\/docs\/build-with-claude\/prompt-engineering\/prompt-templates-and-variables"},{"key":"e_1_3_2_1_4_1","unstructured":"Anthropic. 2024. User Guides - Strengthen guardrails. https:\/\/docs.anthropic.com\/en\/docs\/test-and-evaluate\/strengthen-guardrails\/reduce-hallucinations"},{"key":"e_1_3_2_1_5_1","volume-title":"2017 IEEE\/ACM 25th International Conference on Program Comprehension (ICPC). IEEE, 55\u201365","author":"Avidan Eran","year":"2017","unstructured":"Eran Avidan and Dror G Feitelson. 2017. Effects of variable names on comprehension: An empirical study. In 2017 IEEE\/ACM 25th International Conference on Program Comprehension (ICPC). IEEE, 55\u201365."},{"key":"e_1_3_2_1_6_1","volume-title":"A question-entailment approach to question answering. BMC bioinformatics 20","author":"Abacha Asma Ben","year":"2019","unstructured":"Asma Ben Abacha and Dina Demner-Fushman. 2019. A question-entailment approach to question answering. BMC bioinformatics 20 (2019), 1\u201323."},{"key":"e_1_3_2_1_7_1","unstructured":"Diego Carpintero. 2024. wikisearch. https:\/\/github.com\/dcarpintero\/wikisearch"},{"key":"e_1_3_2_1_8_1","volume-title":"Honest AI: Fine-Tuning \"Small\" Language Models to Say \"I Don't Know\", and Reducing Hallucination in RAG. ArXiv abs\/2410.09699","author":"Chen Xinxi","year":"2024","unstructured":"Xinxi Chen, Li Wang, Wei Wu, Qizhi Tang, and Yiyao Liu. 2024. Honest AI: Fine-Tuning \"Small\" Language Models to Say \"I Don't Know\", and Reducing Hallucination in RAG. ArXiv abs\/2410.09699 (2024). https:\/\/api.semanticscholar.org\/CorpusID:273346023"},{"key":"e_1_3_2_1_9_1","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence","volume":"38","author":"Chen Yihan","year":"2024","unstructured":"Yihan Chen, Benfeng Xu, Quan Wang, Yi Liu, and Zhendong Mao. 2024. Benchmarking large language models on controllable generation under diversified instructions. In Proceedings of the AAAI Conference on Artificial Intelligence, Vol. 38. 17808\u201317816."},{"key":"e_1_3_2_1_10_1","unstructured":"Google Cloud. 2024. Prompt Design Strategies for Generative AI. https:\/\/cloud.google.com\/vertex-ai\/generative-ai\/docs\/learn\/prompts\/prompt-design-strategies Accessed: 2024-07-14."},{"key":"e_1_3_2_1_11_1","volume-title":"Proceedings of the 13th International Conference on Mining Software Repositories. 137\u2013141","author":"Cosentino Valerio","year":"2016","unstructured":"Valerio Cosentino, Javier Luis, and Jordi Cabot. 2016. Findings from GitHub: methods, datasets and limitations. In Proceedings of the 13th International Conference on Mining Software Repositories. 137\u2013141."},{"key":"e_1_3_2_1_12_1","volume-title":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval. 719\u2013729","author":"Cuconasu Florin","year":"2024","unstructured":"Florin Cuconasu, Giovanni Trappolini, Federico Siciliano, Simone Filice, Cesare Campagnano, Yoelle Maarek, Nicola Tonellotto, and Fabrizio Silvestri. 2024. The power of noise: Redefining retrieval for rag systems. In Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval. 719\u2013729."},{"key":"e_1_3_2_1_13_1","unstructured":"Ross Dawson. 2024. Humans + AI: Prompt Elements. https:\/\/rossdawson.com\/humans-plus-ai-old\/humans-ai-prompt_elements\/ Accessed: 2024-10-12."},{"key":"e_1_3_2_1_14_1","unstructured":"Definitive. 2024. openassistants. https:\/\/github.com\/definitive-io\/openassistants"},{"key":"e_1_3_2_1_15_1","volume-title":"Proceedings 2024 Network and Distributed System Security Symposium","author":"Deng Gelei","year":"2023","unstructured":"Gelei Deng, Yi Liu, Yuekang Li, Kailong Wang, Ying Zhang, Zefeng Li, Haoyu Wang, Tianwei Zhang, and Yang Liu. 2023. MASTERKEY: Automated Jailbreaking of Large Language Model Chatbots. Proceedings 2024 Network and Distributed System Security Symposium (2023). https:\/\/api.semanticscholar.org\/CorpusID:259951184"},{"key":"e_1_3_2_1_16_1","volume-title":"Exploring Prompt Engineering Practices in the Enterprise. arXiv preprint arXiv:2403.08950","author":"Desmond Michael","year":"2024","unstructured":"Michael Desmond and Michelle Brachman. 2024. Exploring Prompt Engineering Practices in the Enterprise. arXiv preprint arXiv:2403.08950 (2024)."},{"key":"e_1_3_2_1_17_1","volume-title":"Attention in natural language processing","author":"Galassi Andrea","year":"2020","unstructured":"Andrea Galassi, Marco Lippi, and Paolo Torroni. 2020. Attention in natural language processing. IEEE transactions on neural networks and learning systems 32, 10 (2020), 4291\u20134308."},{"key":"e_1_3_2_1_18_1","volume-title":"Mart: Improving llm safety with multi-round automatic red-teaming. arXiv preprint arXiv:2311.07689","author":"Ge Suyu","year":"2023","unstructured":"Suyu Ge, Chunting Zhou, Rui Hou, Madian Khabsa, Yi-Chia Wang, Qifan Wang, Jiawei Han, and Yuning Mao. 2023. Mart: Improving llm safety with multi-round automatic red-teaming. arXiv preprint arXiv:2311.07689 (2023)."},{"key":"e_1_3_2_1_19_1","unstructured":"Google Cloud. 2024. Generative AI on Vertex AI - Use prompt templates. https:\/\/cloud.google.com\/vertex-ai\/generative-ai\/docs\/learn\/prompts\/prompt-templates"},{"key":"e_1_3_2_1_20_1","volume-title":"International Conference on Human-Centred Software Engineering. Springer, 243\u2013254","author":"Gorissen Simon Cornelius","year":"2024","unstructured":"Simon Cornelius Gorissen, Stefan Sauer, and Wolf G Beckmann. 2024. A Survey of Natural Language-Based Editing of Low-Code Applications Using Large Language Models. In International Conference on Human-Centred Software Engineering. Springer, 243\u2013254."},{"key":"e_1_3_2_1_21_1","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence","volume":"38","author":"Goyal Shubh","year":"2024","unstructured":"Shubh Goyal, Medha Hira, Shubham Mishra, Sukriti Goyal, Arnav Goel, Niharika Dadu, DB Kirushikesh, Sameep Mehta, and Nishtha Madaan. 2024. LLMGuard: Guarding against Unsafe LLM Behavior. In Proceedings of the AAAI Conference on Artificial Intelligence, Vol. 38. 23790\u201323792."},{"key":"e_1_3_2_1_22_1","volume-title":"Proceedings of the 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis. 1073\u20131085","author":"Guo Lianghong","year":"2024","unstructured":"Lianghong Guo, Yanlin Wang, Ensheng Shi, Wanjun Zhong, Hongyu Zhang, Jiachi Chen, Ruikai Zhang, Yuchi Ma, and Zibin Zheng. 2024. When to stop? towards efficient code generation in llms with excess token prevention. In Proceedings of the 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis. 1073\u20131085."},{"key":"e_1_3_2_1_23_1","unstructured":"hambuger. 2023. Andrew. https:\/\/github.com\/hambuger\/Andrew"},{"key":"e_1_3_2_1_24_1","volume-title":"Proceedings of the 2024 ACM SIGPLAN International Symposium on New Ideas, New Paradigms, and Reflections on Programming and Software. 130\u2013146","author":"Holter Karoliine","year":"2024","unstructured":"Karoliine Holter, Juhan Oskar Hennoste, Patrick Lam, Simmo Saan, and Vesal Vojdani. 2024. Abstract debuggers: Exploring program behaviors using static analysis results. In Proceedings of the 2024 ACM SIGPLAN International Symposium on New Ideas, New Paradigms, and Reflections on Programming and Software. 130\u2013146."},{"key":"e_1_3_2_1_25_1","unstructured":"Matthew Honnibal and Ines Montani. 2017. spaCy 2: Natural language understanding with Bloom embeddings convolutional neural networks and incremental parsing. (2017). To appear."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3695988","article-title":"Large language models for software engineering: A systematic literature review","volume":"33","author":"Hou Xinyi","year":"2024","unstructured":"Xinyi Hou, Yanjie Zhao, Yue Liu, Zhou Yang, Kailong Wang, Li Li, Xiapu Luo, David Lo, John Grundy, and Haoyu Wang. 2024. Large language models for software engineering: A systematic literature review. ACM Transactions on Software Engineering and Methodology 33, 8 (2024), 1\u201379.","journal-title":"ACM Transactions on Software Engineering and Methodology"},{"key":"e_1_3_2_1_27_1","volume-title":"On the (In) Security of LLM App Stores. arXiv preprint arXiv:2407.08422","author":"Hou Xinyi","year":"2024","unstructured":"Xinyi Hou, Yanjie Zhao, and Haoyu Wang. 2024. On the (In) Security of LLM App Stores. arXiv preprint arXiv:2407.08422 (2024)."},{"key":"e_1_3_2_1_28_1","unstructured":"Hypercerts. 2024. hypercerts. https:\/\/github.com\/hypercerts-org\/hypercerts"},{"key":"e_1_3_2_1_29_1","unstructured":"International Organization for Standardization. 2019. ISO 8601. https:\/\/www.iso.org\/iso-8601-date-and-time-format.html"},{"key":"e_1_3_2_1_30_1","unstructured":"Jina AI. 2023. dev-gpt. https:\/\/github.com\/jina-ai\/dev-gpt"},{"key":"e_1_3_2_1_31_1","volume-title":"CoPrompter: User-Centric Evaluation of LLM Instruction Alignment for Improved Prompt Engineering. ArXiv abs\/2411.06099","author":"Joshi Ishika","year":"2024","unstructured":"Ishika Joshi, Simra Shahid, Shreeya Venneti, Manushree Vasu, Yantao Zheng, Yunyao Li, Balaji Krishnamurthy, and Gromit Yeuk-Yin Chan. 2024. CoPrompter: User-Centric Evaluation of LLM Instruction Alignment for Improved Prompt Engineering. ArXiv abs\/2411.06099 (2024). https:\/\/api.semanticscholar.org\/CorpusID:273963317"},{"key":"e_1_3_2_1_32_1","volume-title":"TELeR: A General Taxonomy of LLM Prompts for Benchmarking Complex Tasks. ArXiv abs\/2305.11430","author":"Karmaker Shubhra","year":"2023","unstructured":"Shubhra (Santu) Karmaker and Dongji Feng. 2023. TELeR: A General Taxonomy of LLM Prompts for Benchmarking Complex Tasks. ArXiv abs\/2305.11430 (2023). https:\/\/api.semanticscholar.org\/CorpusID:258823169"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"crossref","first-page":"905","DOI":"10.1007\/s00778-022-00776-8","article-title":"A survey on deep learning approaches for text-to-SQL","volume":"32","author":"Katsogiannis-Meimarakis George","year":"2023","unstructured":"George Katsogiannis-Meimarakis and Georgia Koutrika. 2023. A survey on deep learning approaches for text-to-SQL. The VLDB Journal 32, 4 (2023), 905\u2013936.","journal-title":"The VLDB Journal"},{"key":"e_1_3_2_1_34_1","unstructured":"kenoharada. 2023. AI-LaBuddy. https:\/\/github.com\/kenoharada\/AI-LaBuddy"},{"key":"e_1_3_2_1_35_1","volume-title":"Proceedings of the ACM on Programming Languages 8, PLDI","author":"Ketkar Ameya","year":"2024","unstructured":"Ameya Ketkar, Daniel Ramos, Lazaro Clapp, Raj Barik, and Murali Krishna Ramanathan. 2024. A Lightweight Polyglot Code Transformation Language. Proceedings of the ACM on Programming Languages 8, PLDI (2024), 1288\u20131312."},{"key":"e_1_3_2_1_36_1","unstructured":"Kwon Ko. 2024. chart-llm. https:\/\/github.com\/hyungkwonko\/chart-llm"},{"key":"e_1_3_2_1_37_1","unstructured":"Kyubyong. 2019. A TensorFlow Implementation of Transformer. https:\/\/github.com\/Kyubyong\/transformer"},{"key":"e_1_3_2_1_38_1","unstructured":"LAION-AI. 2024. Open-Assistant. https:\/\/github.com\/LAION-AI\/Open-Assistant"},{"key":"e_1_3_2_1_39_1","unstructured":"LangChain. 2024. Prompt Templates. https:\/\/python.langchain.com\/docs\/concepts\/prompt_templates\/"},{"key":"e_1_3_2_1_40_1","volume-title":"2023 IEEE\/ACM 45th International Conference on Software Engineering (ICSE). IEEE, 2124\u20132135","author":"Li Jia","year":"2023","unstructured":"Jia Li, Yongmin Li, Ge Li, Zhi Jin, Yiyang Hao, and Xing Hu. 2023. Skcoder: A sketch-based approach for automatic code generation. In 2023 IEEE\/ACM 45th International Conference on Software Engineering (ICSE). IEEE, 2124\u20132135."},{"key":"e_1_3_2_1_41_1","volume-title":"2023 IEEE\/ACM 45th International Conference on Software Engineering (ICSE). IEEE, 1238\u20131250","author":"Li Zongjie","year":"2023","unstructured":"Zongjie Li, Chaozheng Wang, Zhibo Liu, Haoxuan Wang, Dong Chen, Shuai Wang, and Cuiyun Gao. 2023. Cctest: Testing and repairing code completion systems. In 2023 IEEE\/ACM 45th International Conference on Software Engineering (ICSE). IEEE, 1238\u20131250."},{"key":"e_1_3_2_1_42_1","unstructured":"Yi Liu Gelei Deng Yuekang Li Kailong Wang Zihao Wang Xiaofeng Wang Tianwei Zhang Yepang Liu Haoyu Wang Yan Zheng et al. 2023. Prompt Injection attack against LLM-integrated Applications. arXiv preprint arXiv:2306.05499 (2023)."},{"key":"e_1_3_2_1_43_1","volume-title":"Instruction position matters in sequence generation with large language models. arXiv preprint arXiv:2308.12097","author":"Liu Yijin","year":"2023","unstructured":"Yijin Liu, Xianfeng Zeng, Fandong Meng, and Jie Zhou. 2023. Instruction position matters in sequence generation with large language models. arXiv preprint arXiv:2308.12097 (2023)."},{"key":"e_1_3_2_1_44_1","first-page":"e1434","article-title":"A critical review of state-of-the-art chatbot designs and applications","volume":"12","author":"Luo Bei","year":"2022","unstructured":"Bei Luo, Raymond YK Lau, Chunping Li, and Yain-Whar Si. 2022. A critical review of state-of-the-art chatbot designs and applications. Wiley Interdisciplinary Reviews: Data Mining and Knowledge Discovery 12, 1 (2022), e1434.","journal-title":"Wiley Interdisciplinary Reviews: Data Mining and Knowledge Discovery"},{"key":"e_1_3_2_1_45_1","unstructured":"Maanvitha Gongalla. 2024. thinkai. https:\/\/github.com\/maanvithag\/thinkai"},{"key":"e_1_3_2_1_46_1","volume-title":"Prompt Middleware: Mapping Prompts for Large Language Models to UI Affordances. ArXiv abs\/2307.01142","author":"Macneil Stephen","year":"2023","unstructured":"Stephen Macneil, Andrew Tran, Joanne Kim, Ziheng Huang, Seth Bernstein, and Dan Mogil. 2023. Prompt Middleware: Mapping Prompts for Large Language Models to UI Affordances. ArXiv abs\/2307.01142 (2023). https:\/\/api.semanticscholar.org\/CorpusID:259316650"},{"key":"e_1_3_2_1_47_1","unstructured":"Matt Nigh. 2023. ChatGPT3-Free-Prompt-List: A free guide for learning to create ChatGPT3 Prompts. https:\/\/github.com\/mattnigh\/ChatGPT3-Free-Prompt-List"},{"key":"e_1_3_2_1_48_1","volume-title":"Label-free concept bottleneck models. arXiv preprint arXiv:2304.06129","author":"Oikarinen Tuomas","year":"2023","unstructured":"Tuomas Oikarinen, Subhro Das, Lam M Nguyen, and Tsui-Wei Weng. 2023. Label-free concept bottleneck models. arXiv preprint arXiv:2304.06129 (2023)."},{"key":"e_1_3_2_1_49_1","unstructured":"OpenAI. 2024. OpenAI Prompt Engineering Best Practices. https:\/\/platform.openai.com\/docs\/guides\/prompt-engineering Accessed: 2024-07-14."},{"key":"e_1_3_2_1_50_1","unstructured":"Oracle. 2024. JDK 22 Documentation. https:\/\/docs.oracle.com\/en\/java\/javase\/22"},{"key":"e_1_3_2_1_51_1","volume-title":"Proceedings of the 1st International Workshop on Large Language Models for Code. 62\u201369","author":"Pister Kaiser","year":"2024","unstructured":"Kaiser Pister, Dhruba Jyoti Paul, Ishan Joshi, and Patrick Brophy. 2024. Prompt-Set: A Programmer's Prompting Dataset. In Proceedings of the 1st International Workshop on Large Language Models for Code. 62\u201369."},{"key":"e_1_3_2_1_52_1","unstructured":"puzzle.today. 2024. GPT Store - Write For Me. https:\/\/gptstore.ai\/gpts\/LJybdvxEb4-write-for-me"},{"key":"e_1_3_2_1_53_1","volume-title":"Taskweaver: A code-first agent framework. arXiv preprint arXiv:2311.17541","author":"Qiao Bo","year":"2023","unstructured":"Bo Qiao, Liqun Li, Xu Zhang, Shilin He, Yu Kang, Chaoyun Zhang, Fangkai Yang, Hang Dong, Jue Zhang, Lu Wang, et al. 2023. Taskweaver: A code-first agent framework. arXiv preprint arXiv:2311.17541 (2023)."},{"key":"e_1_3_2_1_54_1","volume-title":"Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering: Software Engineering in Practice. 221\u2013230","author":"Ramanathan Murali Krishna","year":"2020","unstructured":"Murali Krishna Ramanathan, Lazaro Clapp, Rajkishore Barik, and Manu Sridharan. 2020. Piranha: Reducing feature flag debt at uber. In Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering: Software Engineering in Practice. 221\u2013230."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"crossref","unstructured":"Laria Reynolds and Kyle McDonell. 2021. Prompt programming for large language models: Beyond the few-shot paradigm. In Extended abstracts of the 2021 CHI conference on human factors in computing systems. 1\u20137.","DOI":"10.1145\/3411763.3451760"},{"key":"e_1_3_2_1_56_1","unstructured":"Alok Saboo. 2024. beets-plexsync. https:\/\/github.com\/arsaboo\/beets-plexsync"},{"key":"e_1_3_2_1_57_1","unstructured":"Salesforce. 2024. Prompt Builder - Prompt Template Types. https:\/\/help.salesforce.com\/s\/articleView?id=sf.prompt_builder_standard_template_types.htm&type=5"},{"key":"e_1_3_2_1_58_1","volume-title":"The butterfly effect of altering prompts: How small changes and jailbreaks affect large language model performance. arXiv preprint arXiv:2401.03729","author":"Salinas Abel","year":"2024","unstructured":"Abel Salinas and Fred Morstatter. 2024. The butterfly effect of altering prompts: How small changes and jailbreaks affect large language model performance. arXiv preprint arXiv:2401.03729 (2024)."},{"key":"e_1_3_2_1_59_1","unstructured":"Elvis Saravia et al. 2022. Prompt engineering guide. GitHub. URL: https:\/\/github.com\/dair-ai\/Prompt-Engineering-Guide (2022). Accessed: 2024-06-01."},{"key":"e_1_3_2_1_60_1","unstructured":"Sander Schulhoff Michael Ilie Nishant Balepur Konstantine Kahadze Amanda Liu Chenglei Si Yinheng Li Aayush Gupta HyoJung Han Sevien Schulhoff et al. 2024. The Prompt Report: A Systematic Survey of Prompting Techniques. arXiv preprint arXiv:2406.06608 (2024)."},{"key":"e_1_3_2_1_61_1","volume-title":"Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting. arXiv preprint arXiv:2310.11324","author":"Sclar Melanie","year":"2023","unstructured":"Melanie Sclar, Yejin Choi, Yulia Tsvetkov, and Alane Suhr. 2023. Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting. arXiv preprint arXiv:2310.11324 (2023)."},{"key":"e_1_3_2_1_62_1","unstructured":"Stevenic. 2024. AgentM. https:\/\/github.com\/Stevenic\/agentm-py"},{"key":"e_1_3_2_1_63_1","volume-title":"Llama: Open and efficient foundation language models. arXiv preprint arXiv:2302.13971","author":"Touvron Hugo","year":"2023","unstructured":"Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timoth\u00e9e Lacroix, Baptiste Rozi\u00e8re, Naman Goyal, Eric Hambro, Faisal Azhar, et al. 2023. Llama: Open and efficient foundation language models. arXiv preprint arXiv:2302.13971 (2023)."},{"key":"e_1_3_2_1_64_1","volume-title":"LangGPT: Rethinking Structured Reusable Prompt Design Framework for LLMs from the Programming Language. ArXiv abs\/2402.16929","author":"Wang Ming","year":"2024","unstructured":"Ming Wang, Yuanzhong Liu, Xiaoming Zhang, Songlian Li, Yijie Huang, Chi Zhang, Daling Wang, Shi Feng, and Jigang Li. 2024. LangGPT: Rethinking Structured Reusable Prompt Design Framework for LLMs from the Programming Language. ArXiv abs\/2402.16929 (2024). https:\/\/api.semanticscholar.org\/CorpusID:268032985"},{"key":"e_1_3_2_1_65_1","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence","volume":"38","author":"Wang Yu","year":"2024","unstructured":"Yu Wang, Nedim Lipka, Ryan A Rossi, Alexa Siu, Ruiyi Zhang, and Tyler Derr. 2024. Knowledge graph prompting for multi-document question answering. In Proceedings of the AAAI Conference on Artificial Intelligence, Vol. 38. 19206\u201319214."},{"key":"e_1_3_2_1_66_1","volume-title":"Atharva Naik, David Stap, et al.","author":"Wang Yizhong","year":"2022","unstructured":"Yizhong Wang, Swaroop Mishra, Pegah Alipoormolabashi, Yeganeh Kordi, Amirreza Mirzaei, Anjana Arunkumar, Arjun Ashok, Arut Selvan Dhanasekaran, Atharva Naik, David Stap, et al. 2022. Super-naturalinstructions: Generalization via declarative instructions on 1600+ nlp tasks. arXiv preprint arXiv:2204.07705 (2022)."},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"crossref","first-page":"6707","DOI":"10.1109\/TIP.2020.2993098","article-title":"Interpret neural networks by extracting critical subnetworks","volume":"29","author":"Wang Yulong","year":"2020","unstructured":"Yulong Wang, Hang Su, Bo Zhang, and Xiaolin Hu. 2020. Interpret neural networks by extracting critical subnetworks. IEEE Transactions on Image Processing 29 (2020), 6707\u20136720.","journal-title":"IEEE Transactions on Image Processing"},{"key":"e_1_3_2_1_68_1","unstructured":"weaviate. 2024. Verba. https:\/\/github.com\/weaviate\/Verba"},{"key":"e_1_3_2_1_69_1","volume-title":"2014 IEEE International Conference on Data Mining Workshop. IEEE, 851\u2013855","author":"Weber Simon","year":"2014","unstructured":"Simon Weber and Jiebo Luo. 2014. What makes an open source code popular on git hub?. In 2014 IEEE International Conference on Data Mining Workshop. IEEE, 851\u2013855."},{"key":"e_1_3_2_1_70_1","volume-title":"FOFO: A Benchmark to Evaluate LLMs' Format-Following Capability. arXiv preprint arXiv:2402.18667","author":"Xia Congying","year":"2024","unstructured":"Congying Xia, Chen Xing, Jiangshu Du, Xinyi Yang, Yihao Feng, Ran Xu, Wenpeng Yin, and Caiming Xiong. 2024. FOFO: A Benchmark to Evaluate LLMs' Format-Following Capability. arXiv preprint arXiv:2402.18667 (2024)."},{"key":"e_1_3_2_1_71_1","volume-title":"then prompt: Improving accuracy-efficiency trade-off of llm inference with transferable prompt. arXiv preprint arXiv:2305.11186","author":"Xu Zhaozhuo","year":"2023","unstructured":"Zhaozhuo Xu, Zirui Liu, Beidi Chen, Yuxin Tang, Jue Wang, Kaixiong Zhou, Xia Hu, and Anshumali Shrivastava. 2023. Compress, then prompt: Improving accuracy-efficiency trade-off of llm inference with transferable prompt. arXiv preprint arXiv:2305.11186 (2023)."},{"key":"e_1_3_2_1_72_1","volume-title":"Proceedings of the 39th IEEE\/ACM International Conference on Automated Software Engineering. 1370\u20131382","author":"Yan Chuan","year":"2024","unstructured":"Chuan Yan, Ruomai Ren, Mark Huasong Meng, Liuhuo Wan, Tian Yang Ooi, and Guangdong Bai. 2024. Exploring chatgpt app ecosystem: Distribution, deployment and security. In Proceedings of the 39th IEEE\/ACM International Conference on Automated Software Engineering. 1370\u20131382."},{"key":"e_1_3_2_1_73_1","volume-title":"Just rephrase it! Uncertainty estimation in closed-source language models via multiple rephrased queries. arXiv preprint arXiv:2405.13907","author":"Yang Adam","year":"2024","unstructured":"Adam Yang, Chen Chen, and Konstantinos Pitas. 2024. Just rephrase it! Uncertainty estimation in closed-source language models via multiple rephrased queries. arXiv preprint arXiv:2405.13907 (2024)."},{"key":"e_1_3_2_1_74_1","volume-title":"Proceedings of the ACM on Software Engineering 1, FSE","author":"Yuan Zhiqiang","year":"2024","unstructured":"Zhiqiang Yuan, Mingwei Liu, Shiji Ding, Kaixin Wang, Yixuan Chen, Xin Peng, and Yiling Lou. 2024. Evaluating and improving chatgpt for unit test generation. Proceedings of the ACM on Software Engineering 1, FSE (2024), 1703\u20131726."},{"key":"e_1_3_2_1_75_1","volume-title":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","author":"Zamfirescu-Pereira J.D.","year":"2023","unstructured":"J.D. Zamfirescu-Pereira, Richmond Y. Wong, Bjoern Hartmann, and Qian Yang. 2023. Why Johnny Can't Prompt: How Non-AI Experts Try (and Fail) to Design LLM Prompts. Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems (2023). https:\/\/api.semanticscholar.org\/CorpusID:258217984"},{"key":"e_1_3_2_1_76_1","volume-title":"Llm app store analysis: A vision and roadmap. arXiv preprint arXiv:2404.12737","author":"Zhao Yanjie","year":"2024","unstructured":"Yanjie Zhao, Xinyi Hou, Shenao Wang, and Haoyu Wang. 2024. Llm app store analysis: A vision and roadmap. arXiv preprint arXiv:2404.12737 (2024)."},{"key":"e_1_3_2_1_77_1","volume-title":"International conference on machine learning. PMLR, 12697\u201312706","author":"Zhao Zihao","year":"2021","unstructured":"Zihao Zhao, Eric Wallace, Shi Feng, Dan Klein, and Sameer Singh. 2021. Calibrate before use: Improving few-shot performance of language models. In International conference on machine learning. PMLR, 12697\u201312706."}],"event":{"name":"FSE Companion '25: 33rd ACM International Conference on the Foundations of Software Engineering","location":"Clarion Hotel Trondheim Trondheim Norway","acronym":"FSE Companion '25","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 33rd ACM International Conference on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3696630.3728533","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,28]],"date-time":"2025-07-28T19:13:54Z","timestamp":1753730034000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3696630.3728533"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,23]]},"references-count":77,"alternative-id":["10.1145\/3696630.3728533","10.1145\/3696630"],"URL":"https:\/\/doi.org\/10.1145\/3696630.3728533","relation":{},"subject":[],"published":{"date-parts":[[2025,6,23]]},"assertion":[{"value":"2025-07-28","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}