{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,27]],"date-time":"2026-03-27T16:08:29Z","timestamp":1774627709118,"version":"3.50.1"},"publisher-location":"Cham","reference-count":49,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031808883","type":"print"},{"value":"9783031808890","type":"electronic"}],"license":[{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-3-031-80889-0_17","type":"book-chapter","created":{"date-parts":[[2025,1,24]],"date-time":"2025-01-24T08:43:47Z","timestamp":1737708227000},"page":"239-255","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Software System Testing Assisted by Large Language Models: An Exploratory Study"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6140-1375","authenticated-orcid":false,"given":"Cristian","family":"Augusto","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7544-3901","authenticated-orcid":false,"given":"Jes\u00fas","family":"Mor\u00e1n","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8749-1356","authenticated-orcid":false,"given":"Antonia","family":"Bertolino","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5592-9683","authenticated-orcid":false,"given":"Claudio","family":"de la Riva","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1091-934X","authenticated-orcid":false,"given":"Javier","family":"Tuya","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,1,25]]},"reference":[{"key":"17_CR1","unstructured":"Jones, C., Bergen, B.: Does GPT-4 Pass the Turing Test? (2023). http:\/\/arxiv.org\/abs\/2310.20216"},{"key":"17_CR2","unstructured":"Raman, S.: The rise of AI-powered applications: large language models in modern business (2023). https:\/\/www.computer.org\/publications\/tech-news\/trends\/large-language-models-in-modern-business. Accessed 01 Aug 2024"},{"key":"17_CR3","unstructured":"Minaee, S., et al.: Large language models: a survey (2024). http:\/\/arxiv.org\/abs\/2402.06196"},{"issue":"5","key":"17_CR4","doi-asserted-by":"publisher","first-page":"521","DOI":"10.1049\/CMU2.12284","volume":"16","author":"AS Duggal","year":"2022","unstructured":"Duggal, A.S., et al.: A sequential roadmap to Industry 6.0: exploring future manufacturing trends. IET Commun. 16(5), 521\u2013531 (2022). https:\/\/doi.org\/10.1049\/CMU2.12284","journal-title":"IET Commun."},{"key":"17_CR5","unstructured":"MarketsandMarkets, \u201cLarge Language Model Market Size And Share Report, 2030\u201d (2024). https:\/\/www.grandviewresearch.com\/industry-analysis\/large-language-model-llm-market-report. Accessed 01 Aug 2024"},{"key":"17_CR6","unstructured":"P. M. Research, \u201cGlobal Large Language Model (LLM) Market Size, Share, Growth Drivers, Competitive Analysis, Recent Trends & Developments, and Demand Forecast To 2030\u201d (2024). https:\/\/www.pragmamarketresearch.com\/reports\/121032\/large-language-model-llm-market-size. Accessed 01 Aug 2024"},{"issue":"3","key":"17_CR7","doi-asserted-by":"publisher","first-page":"4","DOI":"10.1109\/MS.2023.3248401","volume":"40","author":"I Ozkaya","year":"2023","unstructured":"Ozkaya, I.: Application of large language models to software engineering tasks: opportunities, risks, and implications. IEEE Softw. 40(3), 4\u20138 (2023). https:\/\/doi.org\/10.1109\/MS.2023.3248401","journal-title":"IEEE Softw."},{"key":"17_CR8","doi-asserted-by":"crossref","unstructured":"Hou, X., et al.: Large language models for software engineering: a systematic literature review, vol. X(December), pp. 1\u201379 (2023). http:\/\/arxiv.org\/abs\/2308.10620","DOI":"10.1145\/3695988"},{"key":"17_CR9","doi-asserted-by":"publisher","unstructured":"Jin, P., et al.: Assess and summarize: improve outage understanding with large language models. In: ESEC\/FSE 2023 \u2013 Proceedings of 31st ACM Jt. Meet. European Software Engineering Conference Symposium Foundation Software Engineering, pp. 1657\u20131668 (2023). https:\/\/doi.org\/10.1145\/3611643.3613891","DOI":"10.1145\/3611643.3613891"},{"key":"17_CR10","doi-asserted-by":"publisher","unstructured":"Sobania, D., et al.: Evaluating explanations for\u00a0software patches generated by\u00a0large language models. In: Lecture Notes Computer Science (including Subser. Lect. Notes Artif. Intell. Lect. Notes Bioinformatics), vol. 14415 LNCS, pp. 147\u2013152 (2024). https:\/\/doi.org\/10.1007\/978-3-031-48796-5_12","DOI":"10.1007\/978-3-031-48796-5_12"},{"key":"17_CR11","unstructured":"Betsy, B., Chris, J., Jennifer, P., Niall Richard, M.: Site Reliability Engineering: How Google Runs Production Systems. O\u2019Reilly Media, Inc., (2016)"},{"issue":"1","key":"17_CR12","doi-asserted-by":"publisher","first-page":"85","DOI":"10.1109\/TSE.2023.3334955","volume":"50","author":"M Schafer","year":"2024","unstructured":"Schafer, M., Nadi, S., Eghbali, A., Tip, F.: An empirical evaluation of using large language models for automated unit test generation. IEEE Trans. Softw. Eng. 50(1), 85\u2013105 (2024). https:\/\/doi.org\/10.1109\/TSE.2023.3334955","journal-title":"IEEE Trans. Softw. Eng."},{"key":"17_CR13","doi-asserted-by":"crossref","unstructured":"Arora, C., Herda, T., Homm, V.: Generating test scenarios from NL requirements using retrieval-augmented LLMs: an industrial study (2024). http:\/\/arxiv.org\/abs\/2404.12772","DOI":"10.1109\/RE59067.2024.00031"},{"key":"17_CR14","doi-asserted-by":"publisher","unstructured":"\u201cISO\/IEC\/IEEE International Standard - Software and systems engineering --Software testing --Part 1:General concepts,\u201d ISO\/IEC\/IEEE 29119\u20131:2022(E), pp. 1\u201360 (2022). https:\/\/doi.org\/10.1109\/IEEESTD.2022.9698145","DOI":"10.1109\/IEEESTD.2022.9698145"},{"key":"17_CR15","doi-asserted-by":"publisher","unstructured":"Liu, Z., et al.: Make LLM a testing expert: bringing human-like interaction to mobile GUI testing via functionality-aware decisions, pp. 1\u201313 (2024). https:\/\/doi.org\/10.1145\/3597503.3639180","DOI":"10.1145\/3597503.3639180"},{"issue":"9","key":"17_CR16","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3560815","volume":"55","author":"P Liu","year":"2023","unstructured":"Liu, P., Yuan, W., Fu, J., Jiang, Z., Hayashi, H., Neubig, G.: Pre-train, prompt, and predict: a systematic survey of prompting methods in natural language processing. ACM Comput. Surv. 55(9), 1\u201346 (2023). https:\/\/doi.org\/10.1145\/3560815","journal-title":"ACM Comput. Surv."},{"key":"17_CR17","doi-asserted-by":"publisher","unstructured":"Rumelhart, D.E., Hinton, G.E., Williams, R.J.: Learning internal representations by error propagation. in readings in cognitive science: a perspective from psychology and artificial intelligence, pp. 399\u2013421 (1985). https:\/\/doi.org\/10.1016\/B978-1-4832-1446-7.50035-2","DOI":"10.1016\/B978-1-4832-1446-7.50035-2"},{"issue":"2","key":"17_CR18","doi-asserted-by":"publisher","first-page":"179","DOI":"10.1016\/0364-0213(90)90002-E","volume":"14","author":"JL Elman","year":"1990","unstructured":"Elman, J.L.: Finding structure in time. Cogn. Sci. 14(2), 179\u2013211 (1990). https:\/\/doi.org\/10.1016\/0364-0213(90)90002-E","journal-title":"Cogn. Sci."},{"key":"17_CR19","unstructured":"Mahoney, M.V.M.: Fast text compression with neural networks. In: Proceedings of AAAI FLAIRS (2000). https:\/\/www.aaai.org\/Papers\/FLAIRS\/2000\/FLAIRS00-044.pdf"},{"key":"17_CR20","unstructured":"Vaswani, A., et al.: Attention is all you need. Adv. Neural Inf. Process. Syst. 2017(Decem), 5999\u20136009 (2017). https:\/\/arxiv.org\/abs\/1706.03762v7. Accessed 22 Jul 22 2024"},{"key":"17_CR21","unstructured":"Chowdhery, A., et al.: PaLM: scaling language modeling with pathways (2022). http:\/\/arxiv.org\/abs\/2204.02311. Accessed 22 Jul 2024"},{"key":"17_CR22","unstructured":"Touvron, H., et al.: LLaMA: open and efficient foundation language models (2023).http:\/\/arxiv.org\/abs\/2302.13971. Accessed 22 July 2024"},{"key":"17_CR23","unstructured":"OpenAI et al., \u201cGPT-4 Technical report\u201d (2023). http:\/\/arxiv.org\/abs\/2303.08774"},{"key":"17_CR24","unstructured":"White, J., et al.: A prompt pattern catalog to enhance prompt engineering with ChatGPT (2023). http:\/\/arxiv.org\/abs\/2302.11382"},{"key":"17_CR25","doi-asserted-by":"publisher","unstructured":"Zamfirescu-Pereira, J.D., Wong, R.Y., Hartmann, B., Yang, Q.: Why Johnny Can\u2019t Prompt: How Non-AI Experts Try (and Fail) to Design LLM Prompts. Conf. Hum. Factors Comput. Syst. - Proc. (2023). https:\/\/doi.org\/10.1145\/3544548.3581388","DOI":"10.1145\/3544548.3581388"},{"key":"17_CR26","unstructured":"Dang, H., Mecke, L., Lehmann, F., Goller, S., Buschek, D.: How to Prompt? Opportunities and challenges of zero- and few-shot learning for human-ai interaction in creative applications of generative models, vol. 1, no. 1. Association for Computing Machinery (2022). http:\/\/arxiv.org\/abs\/2209.01390"},{"key":"17_CR27","unstructured":"Wei, J., et al.: Chain-of-thought prompting elicits reasoning in large language models. Adv. Neural Inf. Process. Syst. 35(NeurIPS), 1\u201314 (2022)"},{"key":"17_CR28","unstructured":"Kojima, T., Reid, M., Gu, S.S.: Large language models are zero-shot reasoners. Adv. Neural Inf. Process. Syst. 35(NeurIPS 2022) (2022)"},{"key":"17_CR29","doi-asserted-by":"publisher","unstructured":"Bach, S.H., et al.: PromptSource: an integrated development environment and repository for natural language prompts. In: Proceedings of Annual Meeting of Association of Computer Linguistics, pp. 93\u2013104 (2022). https:\/\/doi.org\/10.18653\/v1\/2022.acl-demo.9","DOI":"10.18653\/v1\/2022.acl-demo.9"},{"key":"17_CR30","unstructured":"Microsoft, \u201cGitHub Copilot \u00b7 Your AI pair programmer\u201d (2022). https:\/\/github.com\/features\/copilot. Accessed 22 July 2024"},{"key":"17_CR31","unstructured":"Spiess, C., et al.: Calibration and Correctness of Language Models for Code (2024). http:\/\/arxiv.org\/abs\/2402.02047. Accessed 22 July 2024"},{"key":"17_CR32","unstructured":"Liu, F., et al.: Exploring and evaluating hallucinations in LLM-powered code generation (2024). http:\/\/arxiv.org\/abs\/2404.00971. Accessed 22 July 2024"},{"key":"17_CR33","doi-asserted-by":"publisher","unstructured":"Nam, D., Macvean, A., Hellendoorn, V., Vasilescu, B., Myers, B.: Using an LLM to help with code understanding. In: Proceedings of the IEEE\/ACM 46th International Conference on Software Engineering, pp. 1\u201313 (2024). https:\/\/doi.org\/10.1145\/3597503.3639187","DOI":"10.1145\/3597503.3639187"},{"issue":"4","key":"17_CR34","doi-asserted-by":"publisher","first-page":"911","DOI":"10.1109\/TSE.2024.3368208","volume":"50","author":"J Wang","year":"2024","unstructured":"Wang, J., Huang, Y., Chen, C., Liu, Z., Wang, S., Wang, Q.: Software testing with large language models: survey, landscape, and vision. IEEE Trans. Softw. Eng. 50(4), 911\u2013936 (2024). https:\/\/doi.org\/10.1109\/TSE.2024.3368208","journal-title":"IEEE Trans. Softw. Eng."},{"key":"17_CR35","doi-asserted-by":"crossref","unstructured":"Ferrari, A., Abualhaija, S., Arora, C.: Model generation from requirements with LLMs: an exploratory study. In: 2024 IEEE 32st International Requirements Engineering Conference Workshops (REW), pp. 291\u2013300 (2024). http:\/\/arxiv.org\/abs\/2404.06371","DOI":"10.1109\/REW61692.2024.00044"},{"key":"17_CR36","unstructured":"Yang, C., Chen, J., Lin, B., Zhou, J., Wang, Z.: Enhancing LLM-based test generation for hard-to-cover branches via program analysis (2024). http:\/\/arxiv.org\/abs\/2404.04966"},{"key":"17_CR37","doi-asserted-by":"publisher","unstructured":"Yu, S., Fang, C., Ling, Y., Wu, C., Chen, Z.: LLM for test script generation and migration: challenges, capabilities, and opportunities. In: IEEE International Conference on Software Quality Reliability and Security QRS, pp. 206\u2013217 (2023). https:\/\/doi.org\/10.1109\/QRS60937.2023.00029","DOI":"10.1109\/QRS60937.2023.00029"},{"key":"17_CR38","doi-asserted-by":"publisher","unstructured":"Liu, Z., et al.: Fill in the blank: context-aware automated text input generation for mobile GUI testing. In: Proceedings of International Conference on Software Engineering, pp. 1355\u20131367 (2023). https:\/\/doi.org\/10.1109\/ICSE48619.2023.00119","DOI":"10.1109\/ICSE48619.2023.00119"},{"key":"17_CR39","doi-asserted-by":"publisher","unstructured":"Shrestha, S.L., Csallner, C.: SLGPT: using transfer learning to directly generate simulink model files and find bugs in the simulink toolchain. In: ACM International Conference on Proceeding Series, pp. 260\u2013265 (2021). https:\/\/doi.org\/10.1145\/3463274.3463806","DOI":"10.1145\/3463274.3463806"},{"key":"17_CR40","doi-asserted-by":"publisher","unstructured":"D. Zimmermann and A. Koziolek, \u201cAutomating GUI-based Software Testing with GPT-3,\u201d Proc. - 2023 IEEE 16th Int. Conf. Softw. Testing, Verif. Valid. Work. ICSTW 2023, pp. 62\u201365, 2023, https:\/\/doi.org\/10.1109\/ICSTW58534.2023.00022","DOI":"10.1109\/ICSTW58534.2023.00022"},{"key":"17_CR41","unstructured":"Ye, X., Durrett, G.: The unreliability of explanations in few-shot prompting for textual reasoning. Adv. Neural Inf. Process. Syst. 35(NeurIPS), 1\u201315 (2022)"},{"key":"17_CR42","unstructured":"ElasTest EU Project, \u201cFullteaching: A web application to make teaching online easy.\u201d Universidad Rey Juan Carlos (2017). https:\/\/github.com\/elastest\/full-teaching. Accessed 10 Aug 2023"},{"key":"17_CR43","doi-asserted-by":"publisher","unstructured":"Garcia, B., et al.: A proposal to orchestrate test cases. In: Proceedings\u20132018 International Conference on the Quality of Information and Communications Technology, QUATIC 2018, pp. 38\u201346 (2018). https:\/\/doi.org\/10.1109\/QUATIC.2018.00016","DOI":"10.1109\/QUATIC.2018.00016"},{"key":"17_CR44","unstructured":"Fuente P\u00e9rez, P.: FullTeaching\u202f: Aplicaci\u00f3n Web de docencia con videoconferencia (2017)"},{"key":"17_CR45","unstructured":"Augusto, C., Mor\u00e1n, J., de la Riva, C., Tuya, J.: FullTeaching E2E Test Suite (2023). https:\/\/github.com\/giis-uniovi\/retorch-st-fullteaching"},{"key":"17_CR46","unstructured":"Augusto, C., Moran, J., Bertolino, A., De La Riva, C., Tuya, J.: Replication package for \u2018software system testing assisted by large language models: an exploratory study (2024). https:\/\/github.com\/giis-uniovi\/retorch-llm-rp. Accessed 22 Jul 2024)"},{"key":"17_CR47","unstructured":"Krensky, P., et al.: Magic quadrant for data science and machine learning platforms (2020). https:\/\/qads.com.br\/data-analytics\/pdfs\/Gartner%0A2018.pdf"},{"key":"17_CR48","unstructured":"OpenAI, \u201cCheat Sheet: Mastering Temperature and Top_p in ChatGPT API - API - OpenAI Developer Forum,\u201d OpenAI Documentation (2023). https:\/\/community.openai.com\/t\/cheat-sheet-mastering-temperature-and-top-p-in-chatgpt-api\/172683. Accessed 23 July 2024)"},{"key":"17_CR49","doi-asserted-by":"publisher","unstructured":"Wohlin, C., Runeson, P., H\u00f6st, M., Ohlsson, M.C., Regnell, B., Wessl\u00e9n, A.: Experimentation in software engineering, vol. 9783642290 (2012). https:\/\/doi.org\/10.1007\/978-3-642-29044-2","DOI":"10.1007\/978-3-642-29044-2"}],"container-title":["Lecture Notes in Computer Science","Testing Software and Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-80889-0_17","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,1,24]],"date-time":"2025-01-24T08:44:06Z","timestamp":1737708246000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-80889-0_17"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"ISBN":["9783031808883","9783031808890"],"references-count":49,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-80889-0_17","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025]]},"assertion":[{"value":"25 January 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ICTSS","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"IFIP International Conference on Testing Software and Systems","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"London","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"United Kingdom","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"31 October 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2 November 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"36","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"pts2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/conf.researchr.org\/home\/ictss-2024","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}