{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T19:13:10Z","timestamp":1776107590919,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":17,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,14]],"date-time":"2024-04-14T00:00:00Z","timestamp":1713052800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,14]]},"DOI":"10.1145\/3639477.3639720","type":"proceedings-article","created":{"date-parts":[[2024,5,31]],"date-time":"2024-05-31T13:27:26Z","timestamp":1717162046000},"page":"103-105","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["Assessing LLMs for High Stakes Applications"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1114-3851","authenticated-orcid":false,"given":"Shannon K.","family":"Gallagher","sequence":"first","affiliation":[{"name":"Software Engineering Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-1898-8415","authenticated-orcid":false,"given":"Jasmine","family":"Ratchford","sequence":"additional","affiliation":[{"name":"Software Engineering Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-3266-1895","authenticated-orcid":false,"given":"Tyler","family":"Brooks","sequence":"additional","affiliation":[{"name":"Software Engineering Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-9372-0246","authenticated-orcid":false,"given":"Bryan P.","family":"Brown","sequence":"additional","affiliation":[{"name":"Software Engineering Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-5544-3204","authenticated-orcid":false,"given":"Eric","family":"Heim","sequence":"additional","affiliation":[{"name":"Software Engineering Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-2198-8918","authenticated-orcid":false,"given":"William R.","family":"Nichols","sequence":"additional","affiliation":[{"name":"Software Engineering Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1868-5178","authenticated-orcid":false,"given":"Scott","family":"Mcmillan","sequence":"additional","affiliation":[{"name":"Software Engineering Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2528-5316","authenticated-orcid":false,"given":"Swati","family":"Rallapalli","sequence":"additional","affiliation":[{"name":"Software Engineering Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0087-3736","authenticated-orcid":false,"given":"Carol J.","family":"Smith","sequence":"additional","affiliation":[{"name":"Software Engineering Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8047-3091","authenticated-orcid":false,"given":"Nathan","family":"Vanhoudnos","sequence":"additional","affiliation":[{"name":"Software Engineering Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-8962-2836","authenticated-orcid":false,"given":"Nick","family":"Winski","sequence":"additional","affiliation":[{"name":"Software Engineering Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-5440-7664","authenticated-orcid":false,"given":"Andrew O.","family":"Mellinger","sequence":"additional","affiliation":[{"name":"Software Engineering Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]}],"member":"320","published-online":{"date-parts":[[2024,5,31]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1111\/nyas.15007"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP40001.2021.00019"},{"key":"e_1_3_2_1_3_1","unstructured":"Hugging Face. 2023. PEFT. Available online at https:\/\/huggingface.co\/docs\/peft\/index."},{"key":"e_1_3_2_1_4_1","unstructured":"Shannon K. Gallagher Andrew O. Mellinger Jasmine Ratchford Nick Winski Tyler Brooks Eric Heim Nathan VanHoudnos Swati Rallapalli William R. Nichols Bryan Brown Angel McDowell and Hollen Barmer. 2023. A Retrospective in Engineering Large Language Models for National Security. Available online at https:\/\/insights.sei.cmu.edu\/library\/a-retrospective-in-engineering-large-language-models-for-national-security\/."},{"key":"e_1_3_2_1_5_1","volume-title":"Bressem","author":"Han Tianyu","year":"2023","unstructured":"Tianyu Han, Lisa C. Adams, Jens-Michalis Papaioannou, Paul Grundmann, Tom Oberhauser, Alexander L\u00f6ser, Daniel Truhn, and Keno K. Bressem. 2023. MedAlpaca - An Open-Source Collection of Medical Conversational AI Models and Training Data. arXiv:2304.08247 [cs.CL]"},{"key":"e_1_3_2_1_6_1","volume-title":"Psychology of intelligence analysis","author":"Heuer Richards J","unstructured":"Richards J Heuer. 1999. Psychology of intelligence analysis. Center for the Study of Intelligence."},{"key":"e_1_3_2_1_7_1","unstructured":"Edward J. Hu Yelong Shen Phillip Wallis Zeyuan Allen-Zhu Yuanzhi Li Shean Wang Lu Wang and Weizhu Chen. 2021. LoRA: Low-Rank Adaptation of Large Language Models. arXiv:2106.09685 [cs.CL]"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1518\/hfes.46.1.50_30392"},{"key":"e_1_3_2_1_9_1","volume-title":"Rouge: A package for automatic evaluation of summaries. In Text summarization branches out. 74--81.","author":"Lin Chin-Yew","year":"2004","unstructured":"Chin-Yew Lin. 2004. Rouge: A package for automatic evaluation of summaries. In Text summarization branches out. 74--81."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/HCS59251.2023.10254721"},{"key":"e_1_3_2_1_11_1","unstructured":"U.S. Department of Defense. 2023. DOD Announces Establishment of Generative AI Task Force. Available online at https:\/\/www.defense.gov\/News\/Releases\/Release\/Article\/3489803\/dod-announces-establishment-of-generative-ai-task-force\/."},{"key":"e_1_3_2_1_13_1","volume-title":"Proceedings of the 40th annual meeting of the Association for Computational Linguistics. 311--318","author":"Papineni Kishore","year":"2002","unstructured":"Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. Bleu: a method for automatic evaluation of machine translation. In Proceedings of the 40th annual meeting of the Association for Computational Linguistics. 311--318."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"crossref","unstructured":"Samyam Rajbhandari Jeff Rasley Olatunji Ruwase and Yuxiong He. 2020. ZeRO: Memory Optimizations Toward Training Trillion Parameter Models. arXiv:1910.02054 [cs.LG]","DOI":"10.1109\/SC41405.2020.00024"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3406703"},{"key":"e_1_3_2_1_16_1","volume-title":"\u0141 ukasz Kaiser, and Illia Polosukhin","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, \u0141 ukasz Kaiser, and Illia Polosukhin. 2017. Attention is All you Need. In Advances in Neural Information Processing Systems, I. Guyon, U. Von Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett (Eds.), Vol. 30. Curran Associates, Inc. https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2017\/file\/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf"},{"key":"e_1_3_2_1_17_1","unstructured":"Eric J. Wang. 2023. Alpaca-LoRA. Available online at https:\/\/github.com\/tloen\/alpaca-lora."},{"key":"e_1_3_2_1_18_1","unstructured":"Lianmin Zheng Wei-Lin Chiang Ying Sheng Siyuan Zhuang Zhanghao Wu Yonghao Zhuang Zi Lin Zhuohan Li Dacheng Li Eric. P Xing Hao Zhang Joseph E. Gonzalez and Ion Stoica. 2023. Judging LLM-as-a-judge with MT-Bench and Chatbot Arena. arXiv:2306.05685 [cs.CL]"}],"event":{"name":"ICSE-SEIP '24: 46th International Conference on Software Engineering: Software Engineering in Practice","location":"Lisbon Portugal","acronym":"ICSE-SEIP '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"]},"container-title":["Proceedings of the 46th International Conference on Software Engineering: Software Engineering in Practice"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3639477.3639720","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3639477.3639720","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:44:31Z","timestamp":1750290271000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3639477.3639720"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,14]]},"references-count":17,"alternative-id":["10.1145\/3639477.3639720","10.1145\/3639477"],"URL":"https:\/\/doi.org\/10.1145\/3639477.3639720","relation":{},"subject":[],"published":{"date-parts":[[2024,4,14]]},"assertion":[{"value":"2024-05-31","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}