{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,10]],"date-time":"2026-10-10T16:08:09Z","timestamp":1791648489795,"version":"4.3.4"},"reference-count":42,"publisher":"IEEE","license":[{"start":{"date-parts":[[2022,5,1]],"date-time":"2022-05-01T00:00:00Z","timestamp":1651363200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2022,5,1]],"date-time":"2022-05-01T00:00:00Z","timestamp":1651363200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022,5]]},"DOI":"10.1109\/icse-seip55303.2022.9793981","type":"proceedings-article","created":{"date-parts":[[2022,6,17]],"date-time":"2022-06-17T19:35:14Z","timestamp":1655494514000},"page":"263-272","source":"Crossref","is-referenced-by-count":1,"title":["Testing Machine Learning Systems in Industry: An Empirical Study"],"prefix":"10.1109","author":[{"given":"Shuyue","family":"Li","sequence":"first","affiliation":[{"name":"Xi&#x0027;an Jiaotong University,Xi&#x0027;an, Shaanxi,China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiaqi","family":"Guo","sequence":"additional","affiliation":[{"name":"Xi&#x0027;an Jiaotong University,Xi&#x0027;an, Shaanxi,China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jian-Guang","family":"Lou","sequence":"additional","affiliation":[{"name":"Xi&#x0027;an Jiaotong University,Xi&#x0027;an, Shaanxi,China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ming","family":"Fan","sequence":"additional","affiliation":[{"name":"Xi&#x0027;an Jiaotong University,Xi&#x0027;an, Shaanxi,China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ting","family":"Liu","sequence":"additional","affiliation":[{"name":"Xi&#x0027;an Jiaotong University,Xi&#x0027;an, Shaanxi,China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Dongmei","family":"Zhang","sequence":"additional","affiliation":[{"name":"Xi&#x0027;an Jiaotong University,Xi&#x0027;an, Shaanxi,China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330579"},{"key":"ref38","article-title":"How does Machine Learning Change Software Development Practices?","author":"wan","year":"2019","journal-title":"IEEE Transactions on Software Engineering"},{"key":"ref33","author":"serban","year":"2020","journal-title":"Adoption and Effects of Software Engineering Best Practices in Machine Learning"},{"key":"ref32","article-title":"Hidden Technical Debt in Machine Learning Systems","author":"sculley","year":"0","journal-title":"Proceedings of the 28th International Conference on Neural Information Processing Systems"},{"key":"ref31","author":"sculley","year":"2014","journal-title":"Machine Learning The High Interest Credit Card of Technical Debt"},{"key":"ref30","author":"rob","year":"2019","journal-title":"Assuring the machine learning lifecycle Desiderata methods and challenges"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-57959-7"},{"key":"ref36","author":"vaswani","year":"2017","journal-title":"Attention is all you need"},{"key":"ref35","author":"siroker","year":"2013","journal-title":"A\/B Testing The Most Powerful Way to Turn Clicks Into Customers"},{"key":"ref34","author":"settles","year":"2009","journal-title":"Active Learning Literature Survey"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/1287624.1287646"},{"key":"ref40","author":"yu","year":"2019","journal-title":"Interpreting and evaluating neural network robustness"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2016.7739680"},{"key":"ref12","year":"0","journal-title":"Alexa"},{"key":"ref13","author":"johirul islam","year":"2019","journal-title":"What do developers ask about ml libraries? a large-scale study using stack overflow"},{"key":"ref14","author":"kalchbrenner","year":"2014","journal-title":"A convolutional Neural Network for Modelling Sentences"},{"key":"ref15","first-page":"63","author":"kitchenham","year":"2008","journal-title":"Personal Opinion Surveys"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177729694"},{"key":"ref17","article-title":"Counterfactual Fairness","author":"kusner","year":"2017","journal-title":"Advances in Neural IInformation Processing Systems"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1145\/2884781.2884874"},{"key":"ref19","first-page":"120","article-title":"DeepGauge: Multigranularity testing criteria for deep learning systems","author":"ma","year":"0","journal-title":"Proceedings of the 33rd ACM\/IEEE International Conference on Automated Software Engineering"},{"key":"ref28","year":"0","journal-title":"The prototype"},{"key":"ref4","author":"buzhinsky","year":"2020","journal-title":"Metrics and methods for robustness evaluation of neural networks with generative models"},{"key":"ref27","first-page":"771","article-title":"Problems and Opportunities in Training Deep Learning Software Systems: An Analysis of Variance","author":"viet pham","year":"0","journal-title":"Proceedings of the 35th ACM\/IEEE International Conference on Automated Software Engineering"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2017.8258038"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338954"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4419-8339-8_13"},{"key":"ref29","article-title":"Beyond Accuracy: Behavioral Testing of NLP Models with CheckList","author":"tulio ribeiro","year":"0","journal-title":"Proceedings annual meeting of the Association for Computational Linguistics"},{"key":"ref8","year":"0","journal-title":"Google Translate"},{"key":"ref7","year":"0","journal-title":"Rules of Machine Learning"},{"key":"ref2","first-page":"2621","article-title":"Measuring Neural Net Robustness with Constraints","author":"bastani","year":"0","journal-title":"Proceedings of the 30th International Conference on Neural Information Processing Systems"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"ref9","year":"0","journal-title":"Monkey"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1016\/S0079-7421(08)60536-8"},{"key":"ref22","year":"0","journal-title":"NVIDIADIGITS"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2010-343"},{"key":"ref24","year":"0","journal-title":"Online Questionnaire"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2018.2870414"},{"key":"ref23","year":"0","journal-title":"Incident Database"},{"key":"ref41","article-title":"Machine Learning Testing: Survey, Landscapes and Horizons","author":"zhang","year":"2020","journal-title":"IEEE Transactions on Software Engineering"},{"key":"ref26","author":"pezze","year":"2008","journal-title":"Software Testing and Analysis Process Principles and Techniques"},{"key":"ref25","first-page":"1","article-title":"Deep-Xplore: Automated Whitebox Testing of Deep Learning Systems","author":"pei","year":"0","journal-title":"Proc Symposium Operating Systems Principles"}],"event":{"name":"2022 IEEE\/ACM 44th International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)","location":"Pittsburgh, PA, USA","start":{"date-parts":[[2022,5,22]]},"end":{"date-parts":[[2022,5,24]]}},"container-title":["2022 IEEE\/ACM 44th International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/9793838\/9793543\/09793981.pdf?arnumber=9793981","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,7,11]],"date-time":"2022-07-11T20:04:30Z","timestamp":1657569870000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9793981\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,5]]},"references-count":42,"URL":"https:\/\/doi.org\/10.1109\/icse-seip55303.2022.9793981","relation":{},"subject":[],"published":{"date-parts":[[2022,5]]}}}