{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,21]],"date-time":"2026-01-21T08:01:46Z","timestamp":1768982506081,"version":"3.49.0"},"reference-count":24,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,9,8]],"date-time":"2025-09-08T00:00:00Z","timestamp":1757289600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,9,8]],"date-time":"2025-09-08T00:00:00Z","timestamp":1757289600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,9,8]]},"DOI":"10.1109\/scam67354.2025.00015","type":"proceedings-article","created":{"date-parts":[[2025,10,10]],"date-time":"2025-10-10T17:35:14Z","timestamp":1760117714000},"page":"78-88","source":"Crossref","is-referenced-by-count":1,"title":["Testing the Untestable? An Empirical Study on the Testing Process of LLM-Powered Software Systems"],"prefix":"10.1109","author":[{"given":"Cleyton","family":"Magalhaes","sequence":"first","affiliation":[{"name":"UFRPE,Recife,PE,Brazil"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Italo","family":"Santos","sequence":"additional","affiliation":[{"name":"University of Hawai‘i at Mānoa,Honolulu,Hawai\u2018i,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Brody","family":"Stuart-Verner","sequence":"additional","affiliation":[{"name":"University of Calgary,Calgary,Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ronnie","family":"De Souza Santos","sequence":"additional","affiliation":[{"name":"University of Calgary,Calgary,Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-024-10602-0"},{"key":"ref2","first-page":"111","article-title":"An empirical study on usage and perceptions of llms in a software engineering project","volume-title":"Proceedings of the 1st International Workshop on Large Language Models for Code","author":"Rasnayaka"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/3695988"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/3715007"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1145\/3640310.3674093"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1145\/3664812"},{"key":"ref7","first-page":"166","article-title":"(why) is my prompt getting worse? rethinking regression testing for evolving llm apis","volume-title":"Proceedings of the IEEE\/ACM 3rd International Conference on AI Engineering-Software Engineering for AI","author":"Ma"},{"key":"ref8","doi-asserted-by":"crossref","DOI":"10.21203\/rs.3.rs-5702357\/v1","article-title":"From testing to evaluation of nlp and llm systems: An analysis of researchers and practitioners perspectives through systematic literature review and developers\u2019 community platforms mining","author":"Asgari","year":"2024"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2024.3368208"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/ICST60714.2024.00039"},{"key":"ref11","first-page":"1607","article-title":"On the evaluation of large language models in unit test generation","volume-title":"Proceedings of the 39th IEEE\/ACM International Conference on Automated Software Engineering","author":"Yang"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.15388\/LMITT.2024.20"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1145\/3659625"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-demo.13"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1145\/3706598.3713680"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1145\/3706599.3719914"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2025.112452"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1145\/3724117"},{"key":"ref19","first-page":"2024","article-title":"A systematic review of testing and evaluation of healthcare applications of large language models (llms)","author":"Bedi","year":"2024","journal-title":"medRxiv"},{"key":"ref20","article-title":"Empirical standards for software engineering research","author":"Ralph","year":"2020"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-008-9102-8"},{"key":"ref22","volume-title":"Case study research and applications: Design and methods.","author":"Yin","year":"2017"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/ESEM.2011.36"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/WSESE66602.2025.00013"}],"event":{"name":"2025 IEEE International Conference on Source Code Analysis & Manipulation (SCAM)","location":"Auckland, New Zealand","start":{"date-parts":[[2025,9,8]]},"end":{"date-parts":[[2025,9,9]]}},"container-title":["2025 IEEE International Conference on Source Code Analysis & Manipulation (SCAM)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11190090\/11189990\/11190124.pdf?arnumber=11190124","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,10,11]],"date-time":"2025-10-11T06:26:30Z","timestamp":1760163990000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11190124\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,9,8]]},"references-count":24,"URL":"https:\/\/doi.org\/10.1109\/scam67354.2025.00015","relation":{},"subject":[],"published":{"date-parts":[[2025,9,8]]}}}