{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,21]],"date-time":"2026-04-21T14:01:30Z","timestamp":1776780090715,"version":"3.51.2"},"reference-count":65,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,4,27]],"date-time":"2025-04-27T00:00:00Z","timestamp":1745712000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,4,27]],"date-time":"2025-04-27T00:00:00Z","timestamp":1745712000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,4,27]]},"DOI":"10.1109\/icse-seip66354.2025.00051","type":"proceedings-article","created":{"date-parts":[[2025,8,20]],"date-time":"2025-08-20T18:28:59Z","timestamp":1755714539000},"page":"516-527","source":"Crossref","is-referenced-by-count":6,"title":["Beyond the Comfort Zone: Emerging Solutions to Overcome Challenges in Integrating LLMs into Software Products"],"prefix":"10.1109","author":[{"given":"Nadia","family":"Nahar","sequence":"first","affiliation":[{"name":"Carnegie Mellon University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Christian","family":"K\u00e4stner","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jenna","family":"Butler","sequence":"additional","affiliation":[{"name":"Microsoft Research"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Chris","family":"Parnin","sequence":"additional","affiliation":[{"name":"Microsoft Research"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Thomas","family":"Zimmermann.","sequence":"additional","affiliation":[{"name":"Microsoft Research"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Christian","family":"Bird","sequence":"additional","affiliation":[{"name":"Microsoft Research"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","volume-title":"Openai\/chatgpt"},{"key":"ref2","volume-title":"Bing copilot"},{"key":"ref3","volume-title":"Teachmemedical"},{"key":"ref4","volume-title":"Predicate"},{"key":"ref5","volume-title":"Ownyourdata llc"},{"key":"ref6","volume-title":"Paxton ai"},{"key":"ref7","volume-title":"Permio ai"},{"key":"ref8","article-title":"Autolead","journal-title":"autolead.so"},{"key":"ref9","article-title":"Artificial intelligence index report 2024","year":"2024","journal-title":"Stanford HAI"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/3729342"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/saner64311.2025.00039"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/CAIN58948.2023.00034"},{"key":"ref13","author":"Hassan","year":"2024","journal-title":"Re-thinking software engineering in the foundation model era: From task-driven ai copilots to goal-driven ai pair programmers"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3639111"},{"key":"ref15","volume-title":"Kennethanceyer\/awesome-llmops"},{"key":"ref16","volume-title":"tensorchord\/awesome-llmops"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642216"},{"key":"ref18","author":"Mishra","year":"2023","journal-title":"Promptaid: Prompt exploration, perturbation, testing and it-eration using visual analytics for large language models"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2022.3209479"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/VIS54172.2023.00056"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.14778\/3685800.3685835"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-demo.40"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1145\/3654777.3676450"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v39i18.34157"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.4135\/9781506335193.n21"},{"key":"ref26","first-page":"209","article-title":"Ad-vanced mixed method research designs","volume":"209","author":"Creswell","year":"2003","journal-title":"Handbook of mixed methods in social & behavioral research"},{"key":"ref27","volume-title":"Supplementary documents: Beyond the comfort zone: Emerging solutions to overcome challenges in integrating llms into software products","year":"2025"},{"key":"ref28","volume-title":"Machine Learning in Production: From Models to Products","author":"Kastner","year":"2024"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3503564"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3519729"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501825"},{"key":"ref32","first-page":"9459","article-title":"Retrieval-augmented generation for knowledge-intensive nlp tasks","volume":"33","author":"Lewis","year":"2020","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.59287\/icaens.1127"},{"issue":"4","key":"ref34","first-page":"129","article-title":"Generative ai service implementation using llm application architecture: based on rag model and langchain framework","volume":"29","author":"Jeong","year":"2023","journal-title":"Journal of Intelligence and Information Systems"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642335"},{"key":"ref36","volume-title":"Testing language models (and prompts) like we test software","author":"Ribeiro","year":"2023"},{"key":"ref37","volume-title":"Prediction guard"},{"key":"ref38","author":"Sahoo","year":"2024","journal-title":"A systematic survey of prompt engineering in large language models: Techniques and applications"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1016\/j.patter.2025.101260"},{"key":"ref40","author":"Vatsal","year":"2024","journal-title":"A survey of prompt engineering meth-ods in large language models for different nlp tasks"},{"key":"ref41","article-title":"Dspy: Compiling declarative language model calls into state-of-the-art pipelines","volume-title":"The Twelfth International Conference on Learning Representations","author":"Khattab","year":"2024"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.346"},{"key":"ref43","first-page":"4222","article-title":"Automatic prompt optimization with","volume-title":"In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)","author":"Pryzant","year":"2020"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-demo.31"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1145\/3581641.3584059"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1145\/3640543.3645144"},{"key":"ref47","article-title":"Sensemaking interfaces for human evaluation of language model outputs","volume-title":"NeurIPS: In Workshop on Human Evaluation of Generative Models","author":"Gero","year":"2022"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v36i1.2564"},{"key":"ref49","volume-title":"Llm survey report, mlops commu-nity","author":"Brinkmann","year":"2023"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581388"},{"key":"ref51","first-page":"166","article-title":"(why) is my prompt getting worse? rethinking regression testing for evolving llm apis","volume-title":"Proceedings of the IEEE\/ACM 3rd International Conference on AI Engineering-Software Engineering for AI","author":"Ma","year":"2024"},{"key":"ref52","author":"Sclar","year":"2023","journal-title":"Quantifying language models\u2019 sensitivity to spurious features in prompt design or: How i learned to start worrying about prompt formatting"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1177\/1744987120927206"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.4135\/9781526421036831710"},{"key":"ref55","volume-title":"Interviewing as qualitative research: A guide for researchers in education and the social sciences","author":"Seidman","year":"2006"},{"key":"ref56","article-title":"Building effective interviewing skills","author":"Goodman","year":"2005","journal-title":"David A. Tepper School of Business, Carnegie Mellon University"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1177\/1049732305276687"},{"key":"ref58","volume-title":"Research methods in human-computer interaction","author":"Lazar","year":"2017"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/ISESE.2003.1237967"},{"key":"ref60","first-page":"46595","article-title":"Judging llm-as-a-judge with mt-bench and chatbot arena","volume":"36","author":"Zheng","year":"2023","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref61","author":"Kamoi","year":"2024","journal-title":"Evaluating llms at detecting errors in llm responses"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1145\/351240.351266"},{"key":"ref63","volume-title":"Why programs fail: a guide to systematic debugging","author":"Zeller","year":"2009"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1145\/3449081"},{"key":"ref65","volume-title":"Microsoft responsible ai standard, v2","year":"2022"}],"event":{"name":"2025 IEEE\/ACM 47th International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)","location":"Ottawa, ON, Canada","start":{"date-parts":[[2025,4,27]]},"end":{"date-parts":[[2025,5,3]]}},"container-title":["2025 IEEE\/ACM 47th International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11121614\/11121615\/11121725.pdf?arnumber=11121725","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T23:57:35Z","timestamp":1755907055000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11121725\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4,27]]},"references-count":65,"URL":"https:\/\/doi.org\/10.1109\/icse-seip66354.2025.00051","relation":{},"subject":[],"published":{"date-parts":[[2025,4,27]]}}}