{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,12]],"date-time":"2026-06-12T18:05:18Z","timestamp":1781287518176,"version":"3.54.1"},"reference-count":3,"publisher":"Association for Computing Machinery (ACM)","issue":"5","license":[{"start":{"date-parts":[[2023,10,31]],"date-time":"2023-10-31T00:00:00Z","timestamp":1698710400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["Queue"],"published-print":{"date-parts":[[2023,10,31]]},"abstract":"<jats:p>We used differential testing to generate test data to improve diversity of data points in the test dataset and then used mutation testing to check the quality of the test data in terms of diversity. Combining differential and mutation testing in this fashion improves mutation score, a test data quality metric, indicating overall improvement in testing effectiveness and quality of the test data when testing deep learning systems.<\/jats:p>","DOI":"10.1145\/3631340","type":"journal-article","created":{"date-parts":[[2023,12,1]],"date-time":"2023-12-01T03:24:31Z","timestamp":1701401071000},"page":"54-65","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Improving Testing of Deep-learning Systems"],"prefix":"10.1145","volume":"21","author":[{"given":"Harsh","family":"Deokuliar","sequence":"first","affiliation":[{"name":"Pennsylvania State University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Raghvinder S.","family":"Sangwan","sequence":"additional","affiliation":[{"name":"Pennsylvania State University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Youakim","family":"Badr","sequence":"additional","affiliation":[{"name":"Pennsylvania State University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Satish M.","family":"Srinivasan","sequence":"additional","affiliation":[{"name":"Pennsylvania State University"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,11,30]]},"reference":[{"key":"e_1_2_1_1_1","unstructured":"LeCun Y. Cortes C. Burges C. The MNIST database of handwritten digits; http:\/\/yann.lecun.com\/exdb\/mnist\/."},{"key":"e_1_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2018.00021"},{"key":"e_1_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2018.00021"}],"container-title":["Queue"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3631340","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3631340","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:35:52Z","timestamp":1750178152000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3631340"}},"subtitle":["A combination of differential and mutation testing results in better test data."],"short-title":[],"issued":{"date-parts":[[2023,10,31]]},"references-count":3,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2023,10,31]]}},"alternative-id":["10.1145\/3631340"],"URL":"https:\/\/doi.org\/10.1145\/3631340","relation":{},"ISSN":["1542-7730","1542-7749"],"issn-type":[{"value":"1542-7730","type":"print"},{"value":"1542-7749","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,10,31]]},"assertion":[{"value":"2023-11-30","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}