{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,25]],"date-time":"2026-07-25T16:33:24Z","timestamp":1784997204515,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":51,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,10,24]],"date-time":"2024-10-24T00:00:00Z","timestamp":1729728000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100006374","name":"Business Finland","doi-asserted-by":"publisher","award":["6GBridge\/6GSoft"],"award-info":[{"award-number":["6GBridge\/6GSoft"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,10,24]]},"DOI":"10.1145\/3674805.3695401","type":"proceedings-article","created":{"date-parts":[[2024,10,15]],"date-time":"2024-10-15T18:39:24Z","timestamp":1729017564000},"page":"517-527","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":17,"title":["Beyond Words: On Large Language Models Actionability in Mission-Critical Risk Analysis"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8451-3668","authenticated-orcid":false,"given":"Matteo","family":"Esposito","sequence":"first","affiliation":[{"name":"University of Oulu, Finland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-5613-3445","authenticated-orcid":false,"given":"Francesco","family":"Palagiano","sequence":"additional","affiliation":[{"name":"Multitel di Lerede Alessandro & C. sas, Italy"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0511-5133","authenticated-orcid":false,"given":"Valentina","family":"Lenarduzzi","sequence":"additional","affiliation":[{"name":"University of Oulu, Finland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3210-3990","authenticated-orcid":false,"given":"Davide","family":"Taibi","sequence":"additional","affiliation":[{"name":"University of Oulu, Italy"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,10,24]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Badr AlKhamissi Millicent Li Asli Celikyilmaz 2022. A Review on Language Models as Knowledge Bases. arxiv:2204.06031\u00a0[cs.CL]"},{"key":"e_1_3_2_1_2_1","volume-title":"Explainable Artificial Intelligence (XAI): Concepts, taxonomies, opportunities and challenges toward responsible AI. Information fusion 58","author":"Arrieta Alejandro\u00a0Barredo","year":"2020","unstructured":"Alejandro\u00a0Barredo Arrieta, Natalia D\u00edaz-Rodr\u00edguez, Javier Del\u00a0Ser, 2020. Explainable Artificial Intelligence (XAI): Concepts, taxonomies, opportunities and challenges toward responsible AI. Information fusion 58 (2020), 82\u2013115."},{"key":"e_1_3_2_1_3_1","volume-title":"Risk analysis","author":"Aven Terje","unstructured":"Terje Aven. 2015. Risk analysis. John Wiley & Sons."},{"key":"e_1_3_2_1_4_1","volume-title":"The Goal Question Metric Approach. Encyclopedia of Software Engineering","author":"Basili R.","year":"1994","unstructured":"V.\u00a0R. Basili, G. Caldiera, and H.\u00a0D. Rombach. 1994. The Goal Question Metric Approach. Encyclopedia of Software Engineering (1994)."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0377-2217(96)00171-3"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3532682"},{"key":"e_1_3_2_1_7_1","unstructured":"Yupeng Chang Xu Wang Jindong Wang 2023. A Survey on Evaluation of Large Language Models. arxiv:2307.03109\u00a0[cs.CL]"},{"key":"e_1_3_2_1_8_1","volume-title":"Software engineering risk analysis and management","author":"Charette N","unstructured":"Robert\u00a0N Charette. 1989. Software engineering risk analysis and management. McGraw-Hill, Inc."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i14.17505"},{"key":"e_1_3_2_1_10_1","unstructured":"Clusit. [n. d.]. Rapporto 2024 sulla sicurezza ICT in Italia. https:\/\/clusit.it\/wp-content\/uploads\/download\/Rapporto_Clusit_2024_web.pdf. Security Summit."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2013.36"},{"key":"e_1_3_2_1_12_1","unstructured":"Common Criteria. 2022. Common Criteria for Information Technology Security Evaluation Part 5: Pre-defined packages of security requirements CC:2022 Revision 1. Technical Report CCMB-2022-11-005. Common Criteria Paris France."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2220352.2220353"},{"key":"e_1_3_2_1_14_1","volume-title":"An Extensive Comparison of Static Application Security Testing Tools. arXiv preprint arXiv:2403.09219","author":"Esposito Matteo","year":"2024","unstructured":"Matteo Esposito, Valentina Falaschi, and Davide Falessi. 2024. An Extensive Comparison of Static Application Security Testing Tools. arXiv preprint arXiv:2403.09219 (2024)."},{"key":"e_1_3_2_1_15_1","volume-title":"Uncovering the Hidden Risks: The Importance of Predicting Bugginess in Untouched Methods. In 2023 IEEE 23rd International Working Conference on Source Code Analysis and Manipulation (SCAM)","author":"Esposito Matteo","unstructured":"Matteo Esposito and Davide Falessi. 2023. Uncovering the Hidden Risks: The Importance of Predicting Bugginess in Untouched Methods. In 2023 IEEE 23rd International Working Conference on Source Code Analysis and Manipulation (SCAM). IEEE, 277\u2013282."},{"key":"e_1_3_2_1_16_1","volume-title":"VALIDATE: A deep dive into vulnerability prediction datasets. Information and Software Technology","author":"Esposito Matteo","year":"2024","unstructured":"Matteo Esposito and Davide Falessi. 2024. VALIDATE: A deep dive into vulnerability prediction datasets. Information and Software Technology (2024), 107448."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/SCAM59687.2023.00037"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3661167.3661226"},{"key":"e_1_3_2_1_19_1","unstructured":"Directorate-General for the Information\u00a0Society European\u00a0Commission and Media. 1991. Information Technology Security Evaluation Criteria (ITSEC) \u2013 Provisional evaluation criteria: Document COM(90) 314. Technical Report. Publications Office of the European Union Luxembourg."},{"key":"e_1_3_2_1_20_1","volume-title":"Measuring nominal scale agreement among many raters.Psychological bulletin 76, 5","author":"Fleiss L","year":"1971","unstructured":"Joseph\u00a0L Fleiss. 1971. Measuring nominal scale agreement among many raters.Psychological bulletin 76, 5 (1971), 378."},{"key":"e_1_3_2_1_21_1","unstructured":"International\u00a0Organization for Standardization. 2022. ISO\/IEC 27001:2022 - Information Security Management Systems. https:\/\/www.iso.org\/standard\/82875.html"},{"key":"e_1_3_2_1_22_1","volume-title":"Advances in Neural Information Processing Systems, A.\u00a0Oh, T.\u00a0Neumann, A.\u00a0Globerson, K.\u00a0Saenko, M.\u00a0Hardt, and S.\u00a0Levine (Eds.). Vol.\u00a036. Curran Associates","author":"Ge Yingqiang","unstructured":"Yingqiang Ge, Wenyue Hua, Kai Mei, 2023. OpenAGI: When LLM Meets Domain Experts. In Advances in Neural Information Processing Systems, A.\u00a0Oh, T.\u00a0Neumann, A.\u00a0Globerson, K.\u00a0Saenko, M.\u00a0Hardt, and S.\u00a0Levine (Eds.). Vol.\u00a036. Curran Associates, Inc., 5539\u20135568."},{"key":"e_1_3_2_1_23_1","unstructured":"Xinyi Hou Yanjie Zhao Yue Liu 2023. Large Language Models for Software Engineering: A Systematic Literature Review. arxiv:2308.10620\u00a0[cs.SE]"},{"key":"e_1_3_2_1_24_1","unstructured":"Wenlong Huang Pieter Abbeel Deepak Pathak 2022. Language Models as Zero-Shot Planners: Extracting Actionable Knowledge for Embodied Agents. arxiv:2201.07207\u00a0[cs.LG]"},{"key":"e_1_3_2_1_25_1","unstructured":"International Organization for Standardization. 2018. ISO 31000:2018 \u2013 Risk management \u2013 Guidelines."},{"key":"e_1_3_2_1_26_1","volume-title":"Atlas: Few-shot Learning with Retrieval Augmented Language Models. arxiv:2208.03299\u00a0[cs.CL]","author":"Izacard Gautier","year":"2022","unstructured":"Gautier Izacard, Patrick Lewis, Maria Lomeli, 2022. Atlas: Few-shot Learning with Retrieval Augmented Language Models. arxiv:2208.03299\u00a0[cs.CL]"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-emnlp.123"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2023.3319768"},{"key":"e_1_3_2_1_29_1","volume-title":"Software engineering risk management: a method, improvement framework, and empirical evaluation","author":"Jyrki Kontio","unstructured":"Jyrki Kontio 2001. Software engineering risk management: a method, improvement framework, and empirical evaluation. Helsinki University of Technology."},{"key":"e_1_3_2_1_30_1","unstructured":"Himabindu Lakkaraju Dylan Slack Yuxin Chen 2022. Rethinking Explainability as a Dialogue: A Practitioner\u2019s Perspective. arxiv:2202.01875\u00a0[cs.LG]"},{"key":"e_1_3_2_1_31_1","first-page":"9459","article-title":"Retrieval-augmented generation for knowledge-intensive nlp tasks","volume":"33","author":"Lewis Patrick","year":"2020","unstructured":"Patrick Lewis, Ethan Perez, Aleksandra Piktus, 2020. Retrieval-augmented generation for knowledge-intensive nlp tasks. Advances in Neural Information Processing Systems 33 (2020), 9459\u20139474.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_32_1","volume-title":"Generation-augmented retrieval for open-domain question answering. arXiv preprint arXiv:2009.08553","author":"Mao Yuning","year":"2020","unstructured":"Yuning Mao, Pengcheng He, Xiaodong Liu, 2020. Generation-augmented retrieval for open-domain question answering. arXiv preprint arXiv:2009.08553 (2020)."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-43458-7_34"},{"key":"e_1_3_2_1_34_1","unstructured":"National Security Authority. 1995. PCM-ANS TI-002: Security Standard for Military EAD Systems\/Networks. Regulation PCM-ANS TI-002. Presidency of the Council of Ministers Italy."},{"key":"e_1_3_2_1_35_1","article-title":"Large language models as tax attorneys: a case study in legal capabilities emergence","volume":"382","author":"Nay J","year":"2024","unstructured":"John\u00a0J Nay, David Karamardian, Sarah\u00a0B Lawsky, 2024. Large language models as tax attorneys: a case study in legal capabilities emergence. Philosophical Transactions of the Royal Society A 382, 2270 (2024), 20230159.","journal-title":"Philosophical Transactions of the Royal Society A"},{"key":"e_1_3_2_1_36_1","volume-title":"CHATREPORT: Democratizing Sustainability Disclosure Analysis through LLM-based Tools. arxiv:2307.15770\u00a0[cs.CL]","author":"Ni Jingwei","year":"2023","unstructured":"Jingwei Ni, Julia Bingler, Chiara Colesanti-Senni, 2023. CHATREPORT: Democratizing Sustainability Disclosure Analysis through LLM-based Tools. arxiv:2307.15770\u00a0[cs.CL]"},{"key":"e_1_3_2_1_37_1","unstructured":"National\u00a0Institute of Standards and Technology. 2020. NIST Special Publication 1800-25A: Data Integrity - Recovering from Ransomware and Other Destructive Events Volume A: Executive Summary. Technical Report. National Institute of Standards and Technology. https:\/\/www.nccoe.nist.gov\/publication\/1800-25\/VolA\/index.html"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.mrl-1.11"},{"key":"e_1_3_2_1_39_1","volume-title":"Sebastian Baltes","author":"Ralph Paul","year":"2021","unstructured":"Paul Ralph, Nauman bin Ali, Sebastian Baltes, 2021. Empirical Standards for Software Engineering Research. arxiv:2010.03525\u00a0[cs.SE]"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-008-9102-8"},{"key":"e_1_3_2_1_41_1","volume-title":"Breaking the silence: the threats of using llms in software engineering. arXiv preprint arXiv:2312.08055","author":"Sallou June","year":"2023","unstructured":"June Sallou, Thomas Durieux, and Annibale Panichella. 2023. Breaking the silence: the threats of using llms in software engineering. arXiv preprint arXiv:2312.08055 (2023)."},{"key":"e_1_3_2_1_42_1","unstructured":"Rita Sevastjanova and Mennatallah El-Assady. 2022. Beware the Rationalization Trap! When Language Model Explainability Diverges from our Mental Models of Language. arxiv:2207.06897\u00a0[cs.CL]"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1002\/ett.3954"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1093\/ptj\/85.3.257"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3579363"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2004.55"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.2307\/3001968"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4615-4625-2"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3581783.3612817"},{"key":"e_1_3_2_1_50_1","volume-title":"Llm lies: Hallucinations are not bugs, but features as adversarial examples. arXiv preprint arXiv:2310.01469","author":"Yao Jia-Yu","year":"2023","unstructured":"Jia-Yu Yao, Kun-Peng Ning, Zhen-Hui Liu, 2023. Llm lies: Hallucinations are not bugs, but features as adversarial examples. arXiv preprint arXiv:2310.01469 (2023)."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"crossref","unstructured":"Tongxin Yuan Zhiwei He Lingzhong Dong 2024. R-Judge: Benchmarking Safety Risk Awareness for LLM Agents. arxiv:2401.10019\u00a0[cs.CL]","DOI":"10.18653\/v1\/2024.findings-emnlp.79"}],"event":{"name":"ESEM '24: ACM \/ IEEE International Symposium on Empirical Software Engineering and Measurement","location":"Barcelona Spain","acronym":"ESEM '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 18th ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3674805.3695401","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3674805.3695401","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T12:56:19Z","timestamp":1755867379000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3674805.3695401"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,24]]},"references-count":51,"alternative-id":["10.1145\/3674805.3695401","10.1145\/3674805"],"URL":"https:\/\/doi.org\/10.1145\/3674805.3695401","relation":{},"subject":[],"published":{"date-parts":[[2024,10,24]]},"assertion":[{"value":"2024-10-24","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}