{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,14]],"date-time":"2026-05-14T17:14:31Z","timestamp":1778778871168,"version":"3.51.4"},"reference-count":43,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,4,14]],"date-time":"2026-04-14T00:00:00Z","timestamp":1776124800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"funder":[{"DOI":"10.13039\/100014440","name":"Spain Ministry of Science Innovation and Universities","doi-asserted-by":"publisher","award":["PID2021-122270OB-I00"],"award-info":[{"award-number":["PID2021-122270OB-I00"]}],"id":[{"id":"10.13039\/100014440","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100014440","name":"Spain Ministry of Science Innovation and Universities","doi-asserted-by":"publisher","award":["PID2024-155231OB-I00"],"award-info":[{"award-number":["PID2024-155231OB-I00"]}],"id":[{"id":"10.13039\/100014440","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100014440","name":"Spain Ministry of Science Innovation and Universities","doi-asserted-by":"publisher","award":["RED2022-134647-T"],"award-info":[{"award-number":["RED2022-134647-T"]}],"id":[{"id":"10.13039\/100014440","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004837","name":"Ministerio de Ciencia e Innovaci\u00f3n","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100004837","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100012818","name":"Community of Madrid","doi-asserted-by":"publisher","award":["2025\/00097\/001"],"award-info":[{"award-number":["2025\/00097\/001"]}],"id":[{"id":"10.13039\/100012818","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Information and Software Technology"],"published-print":{"date-parts":[[2026,8]]},"DOI":"10.1016\/j.infsof.2026.108150","type":"journal-article","created":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T17:22:42Z","timestamp":1776100962000},"page":"108150","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":0,"special_numbering":"C","title":["Towards metamorphic testing with LLM-based workflows: Metamorphic relation inference and follow-up test case generation"],"prefix":"10.1016","volume":"196","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2084-1558","authenticated-orcid":false,"given":"Pablo C.","family":"Ca\u00f1izares","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8319-4829","authenticated-orcid":false,"given":"Pablo","family":"G\u00f3mez-Abajo","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2818-2278","authenticated-orcid":false,"given":"Esther","family":"Guerra","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9425-6362","authenticated-orcid":false,"given":"Juan","family":"de Lara","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"78","reference":[{"issue":"5","key":"10.1016\/j.infsof.2026.108150_b1","doi-asserted-by":"crossref","first-page":"507","DOI":"10.1109\/TSE.2014.2372785","article-title":"The oracle problem in software testing: A survey","volume":"41","author":"Barr","year":"2015","journal-title":"IEEE Trans. Softw. Eng."},{"issue":"9","key":"10.1016\/j.infsof.2026.108150_b2","doi-asserted-by":"crossref","first-page":"805","DOI":"10.1109\/TSE.2016.2532875","article-title":"A survey on metamorphic testing","volume":"42","author":"Segura","year":"2016","journal-title":"IEEE Trans. Softw. Eng."},{"issue":"4","key":"10.1016\/j.infsof.2026.108150_b3","doi-asserted-by":"crossref","first-page":"465","DOI":"10.1093\/comjnl\/25.4.465","article-title":"On testing non-testable programs","volume":"25","author":"Weyuker","year":"1982","journal-title":"Comput. J."},{"key":"10.1016\/j.infsof.2026.108150_b4","series-title":"Metamorphic Testing: A New Approach for Generating Next Test Cases","author":"Chen","year":"1998"},{"issue":"3","key":"10.1016\/j.infsof.2026.108150_b5","doi-asserted-by":"crossref","first-page":"46","DOI":"10.1109\/MS.2018.2875968","article-title":"Metamorphic testing: Testing the untestable","volume":"37","author":"Segura","year":"2020","journal-title":"IEEE Softw."},{"key":"10.1016\/j.infsof.2026.108150_b6","doi-asserted-by":"crossref","DOI":"10.1016\/j.infsof.2023.107164","article-title":"Automated engineering of domain-specific metamorphic testing environments","volume":"157","author":"G\u00f3mez-Abajo","year":"2023","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.infsof.2026.108150_b7","series-title":"ICSE","first-page":"41","article-title":"GeMTest: A general metamorphic testing framework","author":"Speth","year":"2025"},{"issue":"4","key":"10.1016\/j.infsof.2026.108150_b8","doi-asserted-by":"crossref","first-page":"911","DOI":"10.1109\/TSE.2024.3368208","article-title":"Software testing with large language models: Survey, landscape, and vision","volume":"50","author":"Wang","year":"2024","journal-title":"IEEE Trans. Softw. Eng.","ISSN":"https:\/\/id.crossref.org\/issn\/0098-5589","issn-type":"print"},{"key":"10.1016\/j.infsof.2026.108150_b9","series-title":"SAST","first-page":"15","article-title":"An initial investigation of ChatGPT unit test generation capability","author":"Guilherme","year":"2023"},{"key":"10.1016\/j.infsof.2026.108150_b10","article-title":"An empirical evaluation of using large language models for automated unit test generation","author":"Sch\u00e4fer","year":"2023","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.infsof.2026.108150_b11","series-title":"AST@ICSE","first-page":"54","article-title":"Generating accurate assert statements for unit test cases using pretrained transformers","author":"Tufano","year":"2022"},{"key":"10.1016\/j.infsof.2026.108150_b12","series-title":"ICSE","first-page":"640","article-title":"Explaining software bugs leveraging code structures in neural machine translation","author":"Mahbub","year":"2023"},{"issue":"5","key":"10.1016\/j.infsof.2026.108150_b13","doi-asserted-by":"crossref","first-page":"149:1","DOI":"10.1145\/3708521","article-title":"Metamorphic relation generation: State of the art and research directions","volume":"34","author":"Li","year":"2025","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"10.1016\/j.infsof.2026.108150_b14","series-title":"Automated functional testing of web search engines in the absence of an oracle","author":"Zhou","year":"2007"},{"issue":"7","key":"10.1016\/j.infsof.2026.108150_b15","doi-asserted-by":"crossref","first-page":"1888","DOI":"10.1109\/TSE.2024.3407840","article-title":"GenMorph: Automatically generating metamorphic relations via genetic programming","volume":"50","author":"Ayerdi","year":"2024","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.infsof.2026.108150_b16","series-title":"ICCS","first-page":"321","article-title":"Variable discovery with large language models for metamorphic testing of scientific software","volume":"vol. 14073","author":"Tsigkanos","year":"2023"},{"key":"10.1016\/j.infsof.2026.108150_b17","series-title":"COMPSAC","first-page":"1780","article-title":"Automated metamorphic-relation generation with ChatGPT: An experience report","author":"Zhang","year":"2023"},{"key":"10.1016\/j.infsof.2026.108150_b18","doi-asserted-by":"crossref","DOI":"10.1016\/j.infsof.2025.107828","article-title":"Enhancing autonomous driving simulations: A hybrid metamorphic testing framework with metamorphic relations generated by GPT","volume":"187","author":"Zhang","year":"2025","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.infsof.2026.108150_b19","series-title":"Can ChatGPT advance software testing intelligence? An experience report on metamorphic testing","author":"Luu","year":"2023"},{"key":"10.1016\/j.infsof.2026.108150_b20","series-title":"QUATIC","first-page":"126","article-title":"Towards generating executable metamorphic relations using large language models","volume":"vol. 2178","author":"Shin","year":"2024"},{"key":"10.1016\/j.infsof.2026.108150_b21","series-title":"MET@ICSE 2024","first-page":"10","article-title":"Using category partition to detect metamorphic relations","author":"Pedram","year":"2024"},{"issue":"6","key":"10.1016\/j.infsof.2026.108150_b22","doi-asserted-by":"crossref","first-page":"150","DOI":"10.1145\/3656340","article-title":"MR-Scout: Automated synthesis of metamorphic relations from existing test cases","volume":"33","author":"Xu","year":"2024","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"10.1016\/j.infsof.2026.108150_b23","series-title":"QSIC","first-page":"59","article-title":"A new method for constructing metamorphic relations","author":"Liu","year":"2012"},{"key":"10.1016\/j.infsof.2026.108150_b24","series-title":"MET@ICSE 2022","first-page":"42","article-title":"On the cost-effectiveness of composite metamorphic relations for testing deep learning systems","author":"Arrieta","year":"2022"},{"issue":"12","key":"10.1016\/j.infsof.2026.108150_b25","first-page":"392","article-title":"A genetic algorithm-based approach for composite metamorphic relations construction","volume":"10","author":"Xiang","year":"2019","journal-title":"Inf."},{"issue":"3","key":"10.1016\/j.infsof.2026.108150_b26","doi-asserted-by":"crossref","first-page":"1001","DOI":"10.1109\/TSE.2020.3009698","article-title":"Theoretical and empirical analyses of the effectiveness of metamorphic relation composition","volume":"48","author":"Qiu","year":"2022","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.infsof.2026.108150_b27","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2024.112330","article-title":"Effectiveness of symmetric metamorphic relations on validating the stability of code generation LLM","volume":"222","author":"Chan","year":"2025","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.infsof.2026.108150_b28","series-title":"ICSME","article-title":"Metamorphic testing of large language models for natural language processing","author":"Cho","year":"2025"},{"key":"10.1016\/j.infsof.2026.108150_b29","series-title":"MORTAR: Metamorphic multi-turn testing for LLM-based dialogue systems","author":"Guo","year":"2024"},{"key":"10.1016\/j.infsof.2026.108150_b30","series-title":"ICST","first-page":"117","article-title":"METAL: Metamorphic testing framework for analyzing large-language model qualities","author":"Hyun","year":"2024"},{"key":"10.1016\/j.infsof.2026.108150_b31","series-title":"ICISTM","first-page":"180","article-title":"An efficient metamorphic testing technique using genetic algorithm","volume":"vol. 141","author":"Batra","year":"2011"},{"key":"10.1016\/j.infsof.2026.108150_b32","series-title":"PLDI","first-page":"206","article-title":"Grammar-based whitebox fuzzing","author":"Godefroid","year":"2008"},{"key":"10.1016\/j.infsof.2026.108150_b33","series-title":"ICSE","first-page":"748","article-title":"RESTler: Stateful REST API fuzzing","author":"Atlidakis","year":"2019"},{"key":"10.1016\/j.infsof.2026.108150_b34","series-title":"ICFP","first-page":"268","article-title":"QuickCheck: A lightweight tool for random testing of haskell programs","author":"Claessen","year":"2000"},{"key":"10.1016\/j.infsof.2026.108150_b35","unstructured":"OCL, http:\/\/www.omg.org\/spec\/OCL\/."},{"key":"10.1016\/j.infsof.2026.108150_b36","series-title":"SLE","first-page":"49","article-title":"New ideas: Automated engineering of metamorphic testing environments for domain-specific languages","author":"Ca\u00f1izares","year":"2021"},{"key":"10.1016\/j.infsof.2026.108150_b37","series-title":"MODELS-C","first-page":"65","article-title":"Gotten: A model-driven solution to engineer domain-specific metamorphic testing environments","author":"G\u00f3mez-Abajo","year":"2023"},{"issue":"2","key":"10.1016\/j.infsof.2026.108150_b38","doi-asserted-by":"crossref","first-page":"827","DOI":"10.1109\/TR.2022.3193070","article-title":"Performance-driven metamorphic testing of cyber-physical systems","volume":"72","author":"Ayerdi","year":"2023","journal-title":"IEEE Trans. Reliab."},{"key":"10.1016\/j.infsof.2026.108150_b39","series-title":"MathWorks\/vehicle-purepursuit","author":"MathWorks Student Competitions Team","year":"2026"},{"key":"10.1016\/j.infsof.2026.108150_b40","doi-asserted-by":"crossref","DOI":"10.1016\/j.infsof.2026.108069","article-title":"SAME: A similarity analysis method for evaluating metamorphic relations in testing AI systems","author":"Li","year":"2026","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.infsof.2026.108150_b41","series-title":"ICEI","first-page":"1109","article-title":"Software complexity analysis using halstead metrics","author":"Hariprasad","year":"2017"},{"issue":"7","key":"10.1016\/j.infsof.2026.108150_b42","doi-asserted-by":"crossref","first-page":"484","DOI":"10.1109\/32.538605","article-title":"Elements of style: Analyzing a software design feature with a counterexample detector","volume":"22","author":"Jackson","year":"1996","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.infsof.2026.108150_b43","series-title":"EMNLP (Findings)","first-page":"10776","article-title":"NLP evaluation in trouble: On the need to measure LLM data contamination for each benchmark","author":"Sainz","year":"2023"}],"container-title":["Information and Software Technology"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0950584926001394?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0950584926001394?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,5,14]],"date-time":"2026-05-14T16:41:24Z","timestamp":1778776884000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0950584926001394"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,8]]},"references-count":43,"alternative-id":["S0950584926001394"],"URL":"https:\/\/doi.org\/10.1016\/j.infsof.2026.108150","relation":{},"ISSN":["0950-5849"],"issn-type":[{"value":"0950-5849","type":"print"}],"subject":[],"published":{"date-parts":[[2026,8]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Towards metamorphic testing with LLM-based workflows: Metamorphic relation inference and follow-up test case generation","name":"articletitle","label":"Article Title"},{"value":"Information and Software Technology","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.infsof.2026.108150","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 The Authors. Published by Elsevier B.V.","name":"copyright","label":"Copyright"}],"article-number":"108150"}}