{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,28]],"date-time":"2026-10-28T14:02:41Z","timestamp":1793196161680,"version":"4.5.0"},"reference-count":287,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"1","license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"ERC","award":["741278"],"award-info":[{"award-number":["741278"]}]},{"name":"JSPS KAKENHI","award":["19K24348"],"award-info":[{"award-number":["19K24348"]}]},{"name":"JSPS KAKENHI","award":["19H04086"],"award-info":[{"award-number":["19H04086"]}]},{"name":"Qdai-jump Research Program","award":["01277"],"award-info":[{"award-number":["01277"]}]},{"DOI":"10.13039\/100007065","name":"Nvidia","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100007065","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001381","name":"National Research Foundation Singapore","doi-asserted-by":"publisher","award":["NRF2018NCR-NCR005-0001"],"award-info":[{"award-number":["NRF2018NCR-NCR005-0001"]}],"id":[{"id":"10.13039\/501100001381","id-type":"DOI","asserted-by":"publisher"}]},{"name":"National Satellite of Excellence in Trustworthy Software System","award":["NRF2018NCR-NSOE003-0001"],"award-info":[{"award-number":["NRF2018NCR-NSOE003-0001"]}]},{"name":"NTU","award":["NGF-2019-06-024"],"award-info":[{"award-number":["NGF-2019-06-024"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IIEEE Trans. Software Eng."],"published-print":{"date-parts":[[2022,1,1]]},"DOI":"10.1109\/tse.2019.2962027","type":"journal-article","created":{"date-parts":[[2020,2,18]],"date-time":"2020-02-18T03:21:06Z","timestamp":1581996066000},"page":"1-36","source":"Crossref","is-referenced-by-count":637,"title":["Machine Learning Testing: Survey, Landscapes and Horizons"],"prefix":"10.1109","volume":"48","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0481-7264","authenticated-orcid":false,"given":"Jie M.","family":"Zhang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mark","family":"Harman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8621-2420","authenticated-orcid":false,"given":"Lei","family":"Ma","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7300-9215","authenticated-orcid":false,"given":"Yang","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.312"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2017.07.005"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1017\/9781316771273"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1145\/3106237.3106277"},{"key":"ref6","article-title":"Unit-testing statistical software","author":"Pacula","year":"2011"},{"key":"ref7","first-page":"786","article-title":"Integrating symbolic and statistical methods for testing intelligent systems: Applications to machine learning and computer vision","volume-title":"Proc. Des. Autom. Test Europe Conf. Exhib.","author":"Ramanathan"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2014.2372785"},{"key":"ref10","article-title":"An approach to software testing of machine learning applications","volume-title":"Proc. 19th Int. Conf. Softw. Eng. Knowl. Eng.","author":"Murphy"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1145\/800175.809889"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1145\/2568225.2568314"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/52.382180"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipl.2012.01.004"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/SCAM.2008.36"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/ICSM.1993.366932"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1016\/S0950-5849(01)00189-6"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2009.71"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2002.1023795"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/1095430.1081750"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/SYSCON.2018.8369576"},{"key":"ref22","article-title":"Secure deep learning engineering: A software quality assurance perspective","author":"Ma","year":"2018"},{"key":"ref23","article-title":"Safety and trustworthiness of deep neural networks: A survey","author":"Huang","year":"2018"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW.2018.00061"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-00847-5_39"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2020.110542"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511809682"},{"key":"ref28","volume-title":"Foundations of Machine Learning","author":"Mohri","year":"2012"},{"key":"ref29","article-title":"Automatic differentiation in pytorch","author":"Paszke","year":"2017"},{"key":"ref30","article-title":"TensorFlow: Large-scale machine learning on heterogeneous systems, 2015","author":"Abadi"},{"key":"ref31","first-page":"2825","article-title":"Scikit-learn: Machine learning in python","volume":"12","author":"Pedregosa","year":"2011","journal-title":"J. Mach. Learn. Res."},{"key":"ref32","article-title":"Keras","author":"Chollet","year":"2015"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/2647868.2654889"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1126\/science.aaa8415"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/21.97458"},{"key":"ref36","article-title":"","volume":"4","author":"Neter","year":"1996","journal-title":"Applied Linear Statistical Models"},{"key":"ref37","first-page":"41","article-title":"A comparison of event models for naive bayes text classification","volume-title":"Proc. Workshop Learn. Text Categorization","volume":"752","author":"McCallum"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1038\/natureI4539"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/D14-1181"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2013.6638947"},{"key":"ref41","article-title":"","year":"2010","journal-title":"IEEE Standard Classification for Software Anomalies"},{"key":"ref42","article-title":"Detecting overfitting via adversarial examples","author":"Werpachowski","year":"2019"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2017.2782678"},{"issue":"1","key":"ref44","first-page":"1","article-title":"On the complexity of best-arm identification in multi-armed bandit models","volume":"17","author":"Kaufmann","year":"2016","journal-title":"The J. Mach. Learn. Res."},{"key":"ref45","article-title":"Data validation for machine learning","volume-title":"Proc. 2nd SysML Conf.","author":"Breck"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/MEMCOD.2018.8556962"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v30i1.10237"},{"key":"ref48","first-page":"1027","article-title":"CRADLE: Cross-backend validation to detect and localize bugs in deep learning libraries","volume-title":"Proc. IEEE\/ACM 41st Int. Conf. Softw. Eng.","author":"Qi"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4615-5269-7"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2008.12.005"},{"key":"ref51","volume-title":"Thoughtful Machine Learning: A Test-Driven Approach","author":"Kirk","year":"2014"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1162\/neco.1994.6.5.851"},{"key":"ref53","first-page":"396","article-title":"The rademacher complexity of co-regularized kernel classes","volume-title":"Proc. 11th Int. Conf. Artif. Intell. Statist.","author":"Rosenberg"},{"key":"ref54","article-title":"Perturbed model validation: A new framework to validate model relevance","author":"Zhang","year":"2019"},{"key":"ref55","article-title":"","year":"1990","journal-title":"IEEE Standard Glossary of Software Engineering Terminology"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2012.06.002"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-63387-9_5"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4419-5906-5_752"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-57959-7"},{"key":"ref60","article-title":"California consumer privacy act"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2017.8258006"},{"key":"ref62","article-title":"The measure and mismeasure of fairness: A critical review of fair machine learning","author":"Corbett-Davies","year":"2018"},{"key":"ref63","article-title":"Feedback loop: The civil rights act of 1964 and its progeny","volume":"49","author":"Days","year":"2004","journal-title":"Louis ULJ"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1145\/3233231"},{"key":"ref65","article-title":"Towards a rigorous science of interpretable machine learning","author":"Doshi-Velez","year":"2017"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1145\/3299869.3300073"},{"key":"ref67","article-title":"Explanation and justification in machine learning: A survey","volume-title":"Proc. Workshop Explainable AI","author":"Biran"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2018.07.007"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v38i3.2741"},{"key":"ref70","article-title":"Interpretable machine learning","author":"Molnar","year":"2019"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1145\/2642937.2642994"},{"key":"ref72","first-page":"9827","article-title":"Scalable end-to-end autonomous vehicle testing via rare-event simulation","volume-title":"Proc. 32nd Neural Inf. Process. Syst.","author":"O\u2019Kelly"},{"key":"ref73","article-title":"Verification for machine learning, autonomy, and neural networks survey","author":"Xiang","year":"2018"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1145\/2601248.2601268"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287588"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"ref77","article-title":"Udacity challenge"},{"key":"ref78","first-page":"2672","article-title":"Generative adversarial nets","volume-title":"Proc. 27th Int. Conf. Neural Inf. Process. Syst.","author":"Goodfellow"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"ref80","first-page":"700","article-title":"Unsupervised image-to-image translation networks","volume-title":"Proc. 31st Int. Conf. Neural Inf. Process. Syst.","author":"Liu"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380422"},{"key":"ref82","article-title":"DeepCruiser: Automated guided testing for stateful deep learning systems","author":"Du","year":"2018"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1109\/SCC.2016.18"},{"key":"ref84","article-title":"Testing neural program analyzers","author":"Rabin","year":"2019"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1145\/3290353"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380420"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1002\/stvr.294"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1145\/1276958.1277175"},{"key":"ref89","article-title":"TensorFuzz: Debugging neural networks with coverage-guided fuzzing","author":"Odena","year":"2018"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3264835"},{"key":"ref91","article-title":"Coverage-guided fuzzing for deep neural networks","author":"Xie","year":"2018"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-89960-2_22"},{"key":"ref94","article-title":"Rigorous agent evaluation: An adversarial approach to uncover catastrophic failures","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Uesato"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1145\/3241979"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.1109\/DSN.2019.00025"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2953066"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33016867"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33017136"},{"key":"ref100","article-title":"Metamorphic relation based adversarial attacks on differentiable neural computer","author":"Chan","year":"2018"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238165"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1109\/IVS.2018.8500421"},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.1007\/0-387-25036-0_10"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.1126\/science.220.4598.671"},{"key":"ref105","doi-asserted-by":"publisher","DOI":"10.1145\/360248.360252"},{"key":"ref106","doi-asserted-by":"publisher","DOI":"10.1109\/ICSM.2010.5609672"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2012.02.006"},{"key":"ref108","article-title":"Symbolic execution for deep neural networks","author":"Gopinath","year":"2018"},{"key":"ref109","article-title":"Automated test generation to detect individual discrimination in ai models","author":"Agarwal","year":"2018"},{"key":"ref110","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"ref111","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238172"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.1145\/1292414.1292425"},{"key":"ref113","doi-asserted-by":"publisher","DOI":"10.1109\/APSEC.2016.049"},{"key":"ref114","article-title":"Metamorphic testing: A new approach for generating next test cases","author":"Chen","year":"1998"},{"key":"ref115","first-page":"867","article-title":"Properties of machine learning applications for use in metamorphic testing","volume-title":"Proc. 20th Int. Conf. Softw. Eng. Knowl. Eng.","author":"Murphy"},{"key":"ref116","doi-asserted-by":"publisher","DOI":"10.1109\/MET.2017.2"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2009.19"},{"key":"ref118","doi-asserted-by":"publisher","DOI":"10.1109\/QSIC.2009.26"},{"key":"ref119","doi-asserted-by":"publisher","DOI":"10.1145\/1656274.1656278"},{"key":"ref120","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-74781-1_13"},{"key":"ref121","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213858"},{"key":"ref122","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2019.00022"},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-69456-6_7"},{"key":"ref124","doi-asserted-by":"publisher","DOI":"10.1109\/LRA.2018.2857402"},{"key":"ref125","article-title":"METTLE: A metamorphic testing approach to validating unsupervised machine learning methods","author":"Xie","year":"2018"},{"key":"ref126","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-13651-2_2"},{"key":"ref127","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"ref128","doi-asserted-by":"publisher","DOI":"10.1145\/1572272.1572295"},{"key":"ref129","doi-asserted-by":"publisher","DOI":"10.1109\/ASWEC.2018.00021"},{"key":"ref130","first-page":"38","article-title":"A Monte Carlo method for metamorphic testing of machine translation services","volume-title":"Proc. 3rd Int. Workshop Metamorphic Testing","author":"Pesu"},{"issue":"1","key":"ref131","first-page":"100","article-title":"Differential testing for software","volume":"10","author":"McKeeman","year":"1998","journal-title":"Digit. Tech. J."},{"key":"ref132","doi-asserted-by":"publisher","DOI":"10.1145\/2666356.2594334"},{"key":"ref133","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2019.00078"},{"key":"ref134","first-page":"23","article-title":"The methodology of N-version programming","volume":"3","author":"Avizienis","year":"1995","journal-title":"Softw. Fault Tolerance"},{"key":"ref135","first-page":"384","article-title":"Multiple-implementation testing of supervised learning software","volume-title":"Proc. Workshop Eng. Depend. Secure Mach. Learn. Syst.","author":"Srisakaokul"},{"key":"ref136","doi-asserted-by":"publisher","DOI":"10.1109\/QRS.2018.00031"},{"key":"ref137","article-title":"Evaluating robustness of neural networks with mixed integer programming","author":"Tjeng","year":"2017"},{"key":"ref138","doi-asserted-by":"publisher","DOI":"10.1145\/2090236.2090255"},{"key":"ref139","first-page":"3315","article-title":"Equality of opportunity in supervised learning","volume-title":"Proc. 30th Int. Conf. Neural Inf. Process. Syst.","author":"Hardt"},{"key":"ref140","article-title":"Fairness-aware machine learning: A perspective","author":"Zliobaite","year":"2017"},{"key":"ref141","article-title":"The promise and peril of human evaluation for model interpretability","author":"Herman","year":"2017"},{"key":"ref142","article-title":"Model assertions for debugging machine learning","volume-title":"Proc. MLSys: Workshop Syst. ML Open Source Softw.","author":"Kang"},{"key":"ref143","article-title":"Lifelong machine learning test","volume-title":"Proc. Workshop \u201dBeyond the Turing Test\u201d AAAI Conf. Artif. Intell.","author":"Li"},{"key":"ref144","doi-asserted-by":"publisher","DOI":"10.1145\/2931037.2931038"},{"key":"ref145","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238172"},{"key":"ref146","doi-asserted-by":"publisher","DOI":"10.1109\/DASC.2000.886883"},{"key":"ref147","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-NIER.2019.00030"},{"key":"ref148","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2018.00021"},{"key":"ref149","doi-asserted-by":"publisher","DOI":"10.1109\/SANER.2019.8668044"},{"key":"ref150","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-NIER.2019.00031"},{"key":"ref151","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2010.62"},{"key":"ref152","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2018.00021"},{"key":"ref153","doi-asserted-by":"publisher","DOI":"10.1109\/QRS-C.2018.00032"},{"key":"ref154","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2017.8258038"},{"key":"ref155","doi-asserted-by":"publisher","DOI":"10.1109\/AITest.2019.000-6"},{"key":"ref156","article-title":"A noise-sensitivity-analysis-based test prioritization technique for deep neural networks","author":"Zhang","year":"2019"},{"key":"ref157","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338930"},{"key":"ref158","doi-asserted-by":"publisher","DOI":"10.1145\/3417330"},{"key":"ref159","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2012.22"},{"key":"ref160","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213866"},{"key":"ref161","doi-asserted-by":"publisher","DOI":"10.1109\/DSN.2018.00066"},{"key":"ref162","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3236082"},{"key":"ref163","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338972"},{"key":"ref164","article-title":"TensorFlow debugger: Debugging dataflow graphs for machine learning","volume-title":"Proc. Reliable Mach. Learn. Wild-NIPS Workshop","author":"Cai"},{"key":"ref165","doi-asserted-by":"publisher","DOI":"10.1145\/3183713.3196934"},{"key":"ref166","doi-asserted-by":"publisher","DOI":"10.1145\/3077257.3077271"},{"key":"ref167","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.10633"},{"key":"ref168","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-63387-9_9"},{"key":"ref169","first-page":"400","article-title":"Telemade: A testing framework for learning-based malware detection systems","volume-title":"Proc. 32nd AAAI Conf. Artif. Intell. Workshops","author":"Yang"},{"key":"ref170","article-title":"Systematic testing of convolutional neural networks for autonomous driving","author":"Dreossi","year":"2017"},{"key":"ref171","doi-asserted-by":"publisher","DOI":"10.1109\/EuroSP.2017.29"},{"key":"ref172","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW.2018.00060"},{"key":"ref173","doi-asserted-by":"publisher","DOI":"10.1126\/science.aag3311"},{"key":"ref174","first-page":"1137","article-title":"A study of cross-validation and bootstrap for accuracy estimation and model selection","volume-title":"Proc. 14th Int. Joint Conf. Artif. Intell.","volume":"14","author":"Kohavi"},{"key":"ref175","doi-asserted-by":"publisher","DOI":"10.1201\/9780429246593"},{"key":"ref176","first-page":"6","article-title":"Why question machine learning evaluation methods","volume-title":"Proc. AAAI Workshop Eval. Methods Mach. Learn.","author":"Japkowicz"},{"key":"ref177","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2011.12.024"},{"key":"ref178","doi-asserted-by":"publisher","DOI":"10.1186\/1471-2288-13-98"},{"key":"ref179","article-title":"The data linter: Lightweight, automated sanity checking for ML data sets","author":"Hynes","year":"2017"},{"key":"ref180","article-title":"BoostClean: Automated error detection and repair for machine learning","author":"Krishnan","year":"2017"},{"key":"ref181","doi-asserted-by":"publisher","DOI":"10.14778\/3229863.3229867"},{"key":"ref182","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3098021"},{"key":"ref183","doi-asserted-by":"publisher","DOI":"10.1021\/ci0342472"},{"key":"ref184","doi-asserted-by":"publisher","DOI":"10.1118\/1.598805"},{"key":"ref185","doi-asserted-by":"publisher","DOI":"10.1118\/1.599017"},{"key":"ref186","doi-asserted-by":"publisher","DOI":"10.1109\/34.31448"},{"key":"ref187","doi-asserted-by":"publisher","DOI":"10.1117\/12.2293818"},{"key":"ref188","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.282"},{"key":"ref189","first-page":"2613","article-title":"Measuring neural net robustness with constraints","volume-title":"Proc. 30th Int. Conf. Neural Inf. Process. Syst.","author":"Bastani"},{"key":"ref190","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2017.49"},{"key":"ref191","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2018\/368"},{"key":"ref192","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01090-4_1"},{"key":"ref193","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-NIER.2019.00032"},{"key":"ref194","doi-asserted-by":"publisher","DOI":"10.1109\/DSN-S.2019.00018"},{"key":"ref195","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2016.41"},{"key":"ref196","article-title":"cleverhans v1.0.0: An adversarial machine learning library","author":"Papernot","year":"2016"},{"key":"ref197","article-title":"Technical report on the cleverhans v2.1.0 adversarial examples library","author":"Papernot","year":"2018"},{"key":"ref198","doi-asserted-by":"publisher","DOI":"10.1109\/DSN-W.2018.00027"},{"key":"ref199","article-title":"Kayotee: A fault injection-based system to assess the safety and reliability of autonomous vehicles to faults and errors","volume-title":"Proc. 3rd IEEE Int. Workshop Automotive Rel. Test","author":"Jha"},{"key":"ref200","article-title":"Towards testing of deep learning systems with training set reduction","author":"Spieker","year":"2019"},{"key":"ref201","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.2477899"},{"key":"ref202","article-title":"On formalizing fairness in prediction with machine learning","author":"Gajane","year":"2017"},{"key":"ref203","doi-asserted-by":"publisher","DOI":"10.1145\/3194770.3194776"},{"key":"ref204","doi-asserted-by":"publisher","DOI":"10.1145\/3306618.3314248"},{"key":"ref205","doi-asserted-by":"publisher","DOI":"10.1109\/RE.2008.61"},{"key":"ref206","first-page":"4066","article-title":"Counterfactual fairness","volume-title":"Proc. 31st Int. Conf. Neural Inf. Process. Syst.","author":"Kusner"},{"key":"ref207","article-title":"The case for process fairness in learning: Feature selection for fair decision making","volume-title":"Proc. NIPS Symp. Mach. Learn. Law","author":"Grgic-Hlaca"},{"key":"ref208","article-title":"Fairness constraints: Mechanisms for fair classification","author":"Zafar","year":"2015"},{"key":"ref209","first-page":"14922","article-title":"Offline contextual bandits with high probability fairness guarantees","volume-title":"Proc. 33rd Annu. Conf. Neural Inf. Process. Syst.","author":"Metevier"},{"key":"ref210","doi-asserted-by":"publisher","DOI":"10.1145\/1772690.1772758"},{"key":"ref211","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-48503-2"},{"key":"ref212","article-title":"A reductions approach to fair classification","author":"Agarwal","year":"2018"},{"key":"ref213","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3264590"},{"key":"ref214","article-title":"Causal testing: Finding defects\u2019 root causes","author":"Johnson","year":"2018","journal-title":"CoRR"},{"key":"ref215","article-title":"Assessing the local interpretability of machine learning models","author":"Friedler","year":"2019","journal-title":"CoRR"},{"key":"ref216","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2018.2876433"},{"key":"ref217","doi-asserted-by":"publisher","DOI":"10.1177\/0962280216661371"},{"key":"ref218","doi-asserted-by":"publisher","DOI":"10.1145\/3243734.3243818"},{"key":"ref219","first-page":"508","article-title":"DD-Finder: Finding differential privacy violations by sampling and optimization","volume-title":"Proc. ACM SIGSAC Conf. Comput. Commun. Secur.","author":"Bichsel"},{"key":"ref220","doi-asserted-by":"publisher","DOI":"10.1145\/3035918.3054782"},{"key":"ref221","article-title":"On detecting adversarial perturbations","author":"Metzen","year":"2017"},{"key":"ref222","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00126"},{"key":"ref223","article-title":"Detecting adversarial samples for deep neural networks through mutation testing","author":"Wang","year":"2018","journal-title":"CoRR"},{"key":"ref224","doi-asserted-by":"publisher","DOI":"10.1145\/3128572.3140444"},{"key":"ref225","doi-asserted-by":"publisher","DOI":"10.14778\/2994509.2994514"},{"key":"ref226","doi-asserted-by":"publisher","DOI":"10.1145\/2882903.2899409"},{"key":"ref227","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2008.17"},{"key":"ref228","article-title":"AlphaClean: Automatic generation of data cleaning pipelines","author":"Krishnan","year":"2019"},{"issue":"4","key":"ref229","first-page":"3","article-title":"Data cleaning: Problems and current approaches","volume":"23","author":"Rahm","year":"2000","journal-title":"IEEE Data Eng. Bull."},{"key":"ref230","volume-title":"TensorFlow Machine Learning Cookbook","author":"McClure","year":"2017"},{"key":"ref231","article-title":"Unit tests for stochastic optimization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Schaul"},{"key":"ref232","doi-asserted-by":"publisher","DOI":"10.1109\/APSEC.2017.41"},{"key":"ref233","article-title":"An orchestrated empirical study on deep learning frameworks and platforms","author":"Ma","year":"2018"},{"key":"ref234","doi-asserted-by":"publisher","DOI":"10.1134\/S0361768818050031"},{"key":"ref235","doi-asserted-by":"publisher","DOI":"10.1145\/3106237.3106256"},{"key":"ref236","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238190"},{"key":"ref237","doi-asserted-by":"publisher","DOI":"10.1145\/3211346.3211349"},{"key":"ref238","doi-asserted-by":"publisher","DOI":"10.1109\/SPW.2018.00027"},{"key":"ref239","article-title":"How to unit test machine learning code","author":"Roberts","year":"2017"},{"key":"ref240","doi-asserted-by":"publisher","DOI":"10.1109\/QRS.2018.00044"},{"key":"ref241","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2010.11.920"},{"key":"ref242","doi-asserted-by":"publisher","DOI":"10.1002\/stvr.308"},{"key":"ref243","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-24855-2_160"},{"key":"ref244","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-26250-1_27"},{"key":"ref245","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180160"},{"key":"ref246","first-page":"63","article-title":"Testing advanced driver assistance systems using multi-objective search and neural networks","volume-title":"Proc. 31st IEEE\/ACM Int. Conf. Autom. Softw. Eng.","author":"Abdessalem"},{"key":"ref247","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238192"},{"key":"ref248","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"ref249","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-Companion.2019.00131"},{"key":"ref250","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-Companion.2019.00131"},{"key":"ref251","doi-asserted-by":"publisher","DOI":"10.1109\/DSN-Industry.2019.00007"},{"key":"ref252","doi-asserted-by":"publisher","DOI":"10.1177\/2053951715622512"},{"key":"ref253","article-title":"MNIST handwritten digit database","author":"LeCun","year":"2010"},{"key":"ref254","article-title":"Fashion-MNIST: A novel image dataset for benchmarking machine learning algorithms","author":"Xiao","year":"2017","journal-title":"CoRR"},{"key":"ref255","article-title":"","author":"Krizhevsky"},{"key":"ref256","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref257","article-title":"Iris data set","author":"Fisher"},{"key":"ref258","article-title":"Reading digits in natural images with unsupervised feature learning","author":"Netzer","year":"2011"},{"key":"ref259","doi-asserted-by":"publisher","DOI":"10.2478\/ausi-2018-0002"},{"key":"ref260","article-title":"Handwritten letters","author":"Belitskaya"},{"key":"ref262","article-title":"DSRC vehicle communications data set","author":"Malebary"},{"key":"ref263","article-title":"The nexar dataset"},{"key":"ref264","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10602-1_48"},{"key":"ref265","doi-asserted-by":"publisher","DOI":"10.5244\/C.31.11"},{"key":"ref266","doi-asserted-by":"publisher","DOI":"10.1177\/0278364913491297"},{"key":"ref267","article-title":"The babi dataset"},{"key":"ref268","article-title":"Visualizing and understanding recurrent networks","author":"Karpathy","year":"2015"},{"key":"ref270","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D15-1075"},{"key":"ref271","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N18-1101"},{"key":"ref273","article-title":"Statlog (german credit data) data set","author":"Hofmann"},{"key":"ref274","article-title":"Adult data set","author":"Kohav"},{"key":"ref275","doi-asserted-by":"publisher","DOI":"10.1016\/j.dss.2014.03.001"},{"key":"ref277","doi-asserted-by":"publisher","DOI":"10.1109\/SSCI.2015.33"},{"key":"ref278","doi-asserted-by":"publisher","DOI":"10.1126\/science.187.4175.398"},{"key":"ref279","article-title":"Data for the propublica story \u2018machine bias\u2019"},{"key":"ref280","doi-asserted-by":"publisher","DOI":"10.1145\/2827872"},{"key":"ref281","article-title":"Zillow prize: Zillow\u2019s home value prediction (Zestimate)"},{"key":"ref282","article-title":"Report to the congress on credit scoring and its effects on the availability and affordability of credit","volume-title":"Board of Governors of the Federal Reserve System","year":"2007"},{"key":"ref283","article-title":"LSAC national longitudinal bar passage study (NLBPS)"},{"key":"ref284","article-title":"Virustotal"},{"key":"ref286","doi-asserted-by":"publisher","DOI":"10.14722\/ndss.2014.23247"},{"key":"ref287","article-title":"UCI chess (king-rook vs. king-pawn) data set","author":"Shapiro","year":"1989"},{"key":"ref288","doi-asserted-by":"publisher","DOI":"10.1109\/SCAM.2018.00009"},{"key":"ref289","doi-asserted-by":"publisher","DOI":"10.1007\/s00766-009-0075-y"},{"key":"ref290","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2019.00033"},{"key":"ref291","doi-asserted-by":"publisher","DOI":"10.1186\/s40537-016-0043-6"},{"key":"ref292","doi-asserted-by":"publisher","DOI":"10.1109\/GCCE.2018.8574766"}],"container-title":["IEEE Transactions on Software Engineering"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/32\/9675297\/09000651.pdf?arnumber=9000651","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,9]],"date-time":"2024-01-09T17:39:30Z","timestamp":1704821970000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9000651\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,1,1]]},"references-count":287,"journal-issue":{"issue":"1"},"URL":"https:\/\/doi.org\/10.1109\/tse.2019.2962027","relation":{},"ISSN":["0098-5589","1939-3520","2326-3881"],"issn-type":[{"value":"0098-5589","type":"print"},{"value":"1939-3520","type":"electronic"},{"value":"2326-3881","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,1,1]]}}}