{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,19]],"date-time":"2026-03-19T01:19:12Z","timestamp":1773883152581,"version":"3.50.1"},"reference-count":45,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"3","license":[{"start":{"date-parts":[[2026,3,1]],"date-time":"2026-03-01T00:00:00Z","timestamp":1772323200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2026,3,1]],"date-time":"2026-03-01T00:00:00Z","timestamp":1772323200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,3,1]],"date-time":"2026-03-01T00:00:00Z","timestamp":1772323200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62372228"],"award-info":[{"award-number":["62372228"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100012226","name":"Fundamental Research Funds for the Central Universities","doi-asserted-by":"publisher","award":["14380029"],"award-info":[{"award-number":["14380029"]}],"id":[{"id":"10.13039\/501100012226","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Open Project of State Key Laboratory for Novel Software Technology at Nanjing University","award":["KFKT2024B21"],"award-info":[{"award-number":["KFKT2024B21"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IIEEE Trans. Software Eng."],"published-print":{"date-parts":[[2026,3]]},"DOI":"10.1109\/tse.2026.3651022","type":"journal-article","created":{"date-parts":[[2026,1,5]],"date-time":"2026-01-05T18:38:37Z","timestamp":1767638317000},"page":"835-854","source":"Crossref","is-referenced-by-count":0,"title":["Deep Learning Framework Testing via Model Mutation: How Far Are We?"],"prefix":"10.1109","volume":"52","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1816-2246","authenticated-orcid":false,"given":"Yanzhou","family":"Mu","sequence":"first","affiliation":[{"name":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-0543-1894","authenticated-orcid":false,"given":"Rong","family":"Wang","sequence":"additional","affiliation":[{"name":"School of Information Science and Technology, Nantong University, Nantong, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5017-8016","authenticated-orcid":false,"given":"Juan","family":"Zhai","sequence":"additional","affiliation":[{"name":"University of Massachusetts Amherst, Amherst, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9930-7111","authenticated-orcid":false,"given":"Chunrong","family":"Fang","sequence":"additional","affiliation":[{"name":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1180-3891","authenticated-orcid":false,"given":"Xiang","family":"Chen","sequence":"additional","affiliation":[{"name":"School of Artificial Intelligence and Computer Science, Nantong University, Nantong, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-8242-9543","authenticated-orcid":false,"given":"Peiran","family":"Yang","sequence":"additional","affiliation":[{"name":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-6810-7608","authenticated-orcid":false,"given":"Zhixiang","family":"Cao","sequence":"additional","affiliation":[{"name":"School of Electronic Science and Engineering, Xi’an Jiaotong University, Xi’an, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-5040-3123","authenticated-orcid":false,"given":"Ruixiang","family":"Qian","sequence":"additional","affiliation":[{"name":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-7651-8582","authenticated-orcid":false,"given":"Shaoyu","family":"Yang","sequence":"additional","affiliation":[{"name":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9592-7022","authenticated-orcid":false,"given":"Zhenyu","family":"Chen","sequence":"additional","affiliation":[{"name":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1056\/NEJMp1606181"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.312"},{"key":"ref3","article-title":"Deep Learning: Adaptive Computation and Machine Learning","author":"Goodfellow","year":"2016"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1038\/nature14539"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338931"},{"key":"ref6","article-title":"teslanews","year":"2023"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2023.111681"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510165"},{"key":"ref9","first-page":"486","article-title":"Audee: Automated testing for deep learning frameworks","volume-title":"Proc. 35th IEEE\/ACM Int. Conf. Automated Softw. Eng.","author":"Guo","year":"2020"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00107"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00037"},{"key":"ref12","doi-asserted-by":"crossref","DOI":"10.1145\/3540250.3549085","article-title":"Fuzzing deep-learning libraries via automated relational API inference","volume-title":"Proc. 30th ACM Joint Eur. Softw. Eng. Conf. Symp. Found. Softw. Eng.","author":"Deng","year":"2022"},{"key":"ref13","first-page":"1174","article-title":"Fuzzing automatic differentiation in deep-learning libraries","volume-title":"Proc. IEEE\/ACM 45th Int. Conf. Softw. Eng. (ICSE)","author":"Yang","year":"2023"},{"key":"ref14","first-page":"995","article-title":"Free lunch for testing: Fuzzing deep-learning libraries from open source","volume-title":"Proc. IEEE\/ACM 44th Int. Conf. Softw. Eng. (ICSE)","author":"Wei","year":"2022"},{"issue":"6","key":"ref15","first-page":"135:1","article-title":"Toward understanding deep learning framework bugs","volume-title":"ACM Trans. Softw. Eng. Method.","volume":"32","author":"Chen","year":"2023"},{"key":"ref16","first-page":"1159","article-title":"An empirical study on program failures of deep learning jobs","volume-title":"Proc. IEEE\/ACM 42nd Int. Conf. Softw. Eng. (ICSE)","author":"Zhang","year":"2020"},{"key":"ref17","first-page":"35","article-title":"An empirical study on performance bugs in deep learning frameworks","volume-title":"Proc. IEEE Int. Conf. Softw. Maintenance Evol. (ICSME)","author":"Makkouk","year":"2022"},{"key":"ref18","article-title":"Silent bugs in deep learning frameworks: An empirical study of Keras and TensorfFlow","author":"Tambon","year":"2021"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409761"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/3583566"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2024.3509975"},{"key":"ref22","article-title":"$\\{${AFL++$\\}$}: Combining incremental steps of fuzzing research","volume-title":"Proc. 14th USENIX Workshop Offensive Technol. (WOOT)","author":"Fioraldi","year":"2020"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330576"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/3460120.3484543"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510092"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1023\/A:1020281327116"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1145\/3628159"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1038\/nature14236"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1145\/3611643.3616337"},{"key":"ref30","article-title":"Tensorflow","year":"2019"},{"key":"ref31","article-title":"Onnx","year":"2019"},{"key":"ref32","article-title":"Pytorch issue report","year":"2018"},{"key":"ref33","article-title":"Mindspore issue report","year":"2020"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1037\/h0031619"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.2307\/3001968"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref38","first-page":"8602","article-title":"Switch transformers: Scaling to trillion parameter models","volume-title":"Proc. 39th Int. Conf. Mach. Learn. (ICML)","author":"Fedus","year":"2021"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1145\/3460319.3464825"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-41714-6_40271"},{"key":"ref41","article-title":"Jittor","year":"2022"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1145\/3460319.3464843"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/TR.2021.3107165"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/MET.2017.2"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1145\/3660796"}],"container-title":["IEEE Transactions on Software Engineering"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/32\/11440029\/11329108.pdf?arnumber=11329108","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,18]],"date-time":"2026-03-18T19:38:45Z","timestamp":1773862725000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11329108\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,3]]},"references-count":45,"journal-issue":{"issue":"3"},"URL":"https:\/\/doi.org\/10.1109\/tse.2026.3651022","relation":{},"ISSN":["0098-5589","1939-3520","2326-3881"],"issn-type":[{"value":"0098-5589","type":"print"},{"value":"1939-3520","type":"electronic"},{"value":"2326-3881","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,3]]}}}