{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,4]],"date-time":"2026-07-04T08:21:10Z","timestamp":1783153270243,"version":"3.54.6"},"reference-count":78,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,5,12]],"date-time":"2026-05-12T00:00:00Z","timestamp":1778544000000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100000780","name":"European Commission","doi-asserted-by":"publisher","award":["D53D23017340001"],"award-info":[{"award-number":["D53D23017340001"]}],"id":[{"id":"10.13039\/501100000780","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000780","name":"European Commission","doi-asserted-by":"publisher","award":["P2022553SL"],"award-info":[{"award-number":["P2022553SL"]}],"id":[{"id":"10.13039\/501100000780","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000780","name":"European Commission","doi-asserted-by":"publisher","award":["D53D23008570006"],"award-info":[{"award-number":["D53D23008570006"]}],"id":[{"id":"10.13039\/501100000780","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100019457","name":"Chips Joint Undertaking","doi-asserted-by":"publisher","award":["KDT232RIA_00017"],"award-info":[{"award-number":["KDT232RIA_00017"]}],"id":[{"id":"10.13039\/100019457","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100024370","name":"Ministero dell'Istruzione dell'Universit\u00e0 e della Ricerca","doi-asserted-by":"publisher","award":["D53D23017340001"],"award-info":[{"award-number":["D53D23017340001"]}],"id":[{"id":"10.13039\/501100024370","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100024370","name":"Ministero dell'Istruzione dell'Universit\u00e0 e della Ricerca","doi-asserted-by":"publisher","award":["D53D23008570006"],"award-info":[{"award-number":["D53D23008570006"]}],"id":[{"id":"10.13039\/501100024370","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100024370","name":"Ministero dell'Istruzione dell'Universit\u00e0 e della Ricerca","doi-asserted-by":"publisher","award":["PE0000013"],"award-info":[{"award-number":["PE0000013"]}],"id":[{"id":"10.13039\/501100024370","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100018705","name":"European Institute of Innovation & Technology","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100018705","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Journal of Systems and Software"],"published-print":{"date-parts":[[2026,10]]},"DOI":"10.1016\/j.jss.2026.112925","type":"journal-article","created":{"date-parts":[[2026,5,12]],"date-time":"2026-05-12T23:07:39Z","timestamp":1778627259000},"page":"112925","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":0,"special_numbering":"C","title":["Understanding Machine Learning testing in practice"],"prefix":"10.1016","volume":"240","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-0209-5974","authenticated-orcid":false,"given":"Alfonso","family":"Cannavale","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6012-9947","authenticated-orcid":false,"given":"Valeria","family":"Pontillo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Andrea","family":"De Lucia","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Fabio","family":"Palomba","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.jss.2026.112925_b1","doi-asserted-by":"crossref","first-page":"131255","DOI":"10.1109\/ACCESS.2024.3458184","article-title":"A large-scale empirical investigation into cross-project flaky test prediction","volume":"12","author":"Afeltra","year":"2024","journal-title":"IEEE Access"},{"key":"10.1016\/j.jss.2026.112925_b2","series-title":"2021 IEEE\/ACM 43rd International Conference on Software Engineering: Companion Proceedings (ICSE-Companion)","first-page":"81","article-title":"Testing framework for black-box AI models","author":"Aggarwal","year":"2021"},{"key":"10.1016\/j.jss.2026.112925_b3","article-title":"Demystifying black-box models with symbolic metamodels","volume":"32","author":"Alaa","year":"2019","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"10.1016\/j.jss.2026.112925_b4","doi-asserted-by":"crossref","unstructured":"Alami, A., Zahedi, M., Ernst, N., 2024. Are you a real software engineer? best practices in online recruitment for software engineering studies. In: Proceedings of the 1st IEEE\/ACM International Workshop on Methodological Issues with Empirical Studies in Software Engineering. pp. 52\u201357.","DOI":"10.1145\/3643664.3648207"},{"key":"10.1016\/j.jss.2026.112925_b5","first-page":"1","article-title":"Testing machine learning and deep learning systems: Achievements and challenges","author":"Albelali","year":"2025","journal-title":"Arab. J. Sci. Eng."},{"key":"10.1016\/j.jss.2026.112925_b6","series-title":"2023 IEEE\/ACM International Conference on Software Engineering: Future of Software Engineering (ICSE-FoSE)","first-page":"4","article-title":"Software testing of generative ai systems: Challenges and opportunities","author":"Aleti","year":"2023"},{"key":"10.1016\/j.jss.2026.112925_b7","series-title":"2019 IEEE\/ACM 41st International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)","first-page":"291","article-title":"Software engineering for machine learning: A case study","author":"Amershi","year":"2019"},{"key":"10.1016\/j.jss.2026.112925_b8","series-title":"Conducting research on the internet: Online survey design, development and implementation guidelines","author":"Andrews","year":"2007"},{"key":"10.1016\/j.jss.2026.112925_b9","series-title":"2021 IEEE\/ACM 43rd International Conference on Software Engineering","first-page":"312","article-title":"Scalable quantitative verification for deep neural networks","author":"Baluta","year":"2021"},{"key":"10.1016\/j.jss.2026.112925_b10","doi-asserted-by":"crossref","unstructured":"Baumann, N., Kusmenko, E., Ritz, J., Rumpe, B., Weber, M.B., 2022. Dynamic data management for continuous retraining. In: Proceedings of the 25th International Conference on Model Driven Engineering Languages and Systems: Companion Proceedings. pp. 359\u2013366.","DOI":"10.1145\/3550356.3561568"},{"key":"10.1016\/j.jss.2026.112925_b11","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2020.110542","article-title":"On testing machine learning programs","volume":"164","author":"Braiek","year":"2020","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.jss.2026.112925_b12","first-page":"1877","article-title":"Language models are few-shot learners","volume":"33","author":"Brown","year":"2020","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"10.1016\/j.jss.2026.112925_b13","doi-asserted-by":"crossref","unstructured":"Camara, B., Silva, M., Endo, A., Vergilio, S., 2021. On the use of test smells for prediction of flaky tests. In: Proceedings of the 6th Brazilian Symposium on Systematic and Automated Software Testing. pp. 46\u201354.","DOI":"10.1145\/3482909.3482916"},{"key":"10.1016\/j.jss.2026.112925_b14","series-title":"Replication package: Understanding machine learning testing in practice","author":"Cannavale","year":"2026"},{"issue":"3","key":"10.1016\/j.jss.2026.112925_b15","first-page":"5","article-title":"Content analysis: concepts, methods and applications.","volume":"4","author":"Cavanagh","year":"1997","journal-title":"Nurse Res."},{"issue":"3","key":"10.1016\/j.jss.2026.112925_b16","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3641289","article-title":"A survey on evaluation of large language models","volume":"15","author":"Chang","year":"2024","journal-title":"ACM Trans. Intell. Syst. Technol."},{"key":"10.1016\/j.jss.2026.112925_b17","series-title":"Handbook on Teaching Empirical Software Engineering","first-page":"325","article-title":"Teaching mining software repositories","author":"Codabux","year":"2024"},{"key":"10.1016\/j.jss.2026.112925_b18","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2025.112471","article-title":"Into the ML-universe: An improved classification and characterization of machine-learning projects","volume":"230","author":"De Martino","year":"2025","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.jss.2026.112925_b19","doi-asserted-by":"crossref","unstructured":"Della Porta, A., Lambiase, S., Palomba, F., 2025. Do prompt patterns affect code quality? a first empirical assessment of chatgpt-generated code. In: Proceedings of the 29th International Conference on Evaluation and Assessment in Software Engineering. pp. 181\u2013192.","DOI":"10.1145\/3756681.3756938"},{"key":"10.1016\/j.jss.2026.112925_b20","unstructured":"Ebert, F., Serebrenik, A., Treude, C., Novielli, N., Castor, F., 2022. On Recruiting Experienced GitHub Contributors for Interviews and Surveys on Prolific. In: International Workshop on Recruiting Participants for Empirical Software Engineering."},{"key":"10.1016\/j.jss.2026.112925_b21","series-title":"International Conference on Software Quality","first-page":"33","article-title":"Quality assurance for AI-based systems: Overview and challenges (introduction to interactive session)","author":"Felderer","year":"2021"},{"key":"10.1016\/j.jss.2026.112925_b22","doi-asserted-by":"crossref","DOI":"10.1016\/j.knosys.2024.112621","article-title":"Similarity-driven adversarial testing of neural networks","volume":"305","author":"Filus","year":"2024","journal-title":"Knowl.-Based Syst."},{"issue":"1","key":"10.1016\/j.jss.2026.112925_b23","doi-asserted-by":"crossref","first-page":"169","DOI":"10.1023\/A:1018955722107","article-title":"On non-determinism in machines and languages","volume":"19","author":"Grumbach","year":"1997","journal-title":"Ann. Math. Artif. Intell."},{"issue":"4","key":"10.1016\/j.jss.2026.112925_b24","doi-asserted-by":"crossref","first-page":"305","DOI":"10.1023\/A:1011922615502","article-title":"Ethical issues in software engineering research: a survey of current practice","volume":"6","author":"Hall","year":"2001","journal-title":"Empir. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112925_b25","doi-asserted-by":"crossref","unstructured":"Harel-Canada, F., Wang, L., Gulzar, M.A., Gu, Q., Kim, M., 2020. Is neuron coverage a meaningful measure for testing deep neural networks?. In: Proceedings of the 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering. pp. 851\u2013862.","DOI":"10.1145\/3368089.3409754"},{"key":"10.1016\/j.jss.2026.112925_b26","series-title":"Econometrics","first-page":"201","article-title":"Selection bias and self-selection","author":"Heckman","year":"1990"},{"key":"10.1016\/j.jss.2026.112925_b27","series-title":"The New Palgrave Dictionary of Economics","first-page":"12130","article-title":"Selection bias and self-selection","author":"Heckman","year":"2018"},{"key":"10.1016\/j.jss.2026.112925_b28","series-title":"2019 34th IEEE\/ACM International Conference on Automated Software Engineering","first-page":"1158","article-title":"Deepmutation++: A mutation testing framework for deep learning systems","author":"Hu","year":"2019"},{"issue":"1","key":"10.1016\/j.jss.2026.112925_b29","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/1471-2288-13-3","article-title":"Participant recruitment in sensitive surveys: a comparative trial of \u2018opt in\u2019versus \u2018opt out\u2019approaches","volume":"13","author":"Hunt","year":"2013","journal-title":"BMC Med. Res. Methodol."},{"key":"10.1016\/j.jss.2026.112925_b30","series-title":"2024 IEEE Conference on Software Testing, Verification and Validation","first-page":"117","article-title":"METAL: metamorphic testing framework for analyzing large-language model qualities","author":"Hyun","year":"2024"},{"key":"10.1016\/j.jss.2026.112925_b31","series-title":"2019 IEEE\/ACM 41st International Conference on Software Engineering","first-page":"1039","article-title":"Guiding deep learning system testing using surprise adequacy","author":"Kim","year":"2019"},{"key":"10.1016\/j.jss.2026.112925_b32","series-title":"2019 IEEE International Conference on Artificial Intelligence Testing (AITest)","first-page":"81","article-title":"AI for testing today and tomorrow: industry perspectives","author":"King","year":"2019"},{"issue":"1","key":"10.1016\/j.jss.2026.112925_b33","doi-asserted-by":"crossref","first-page":"18","DOI":"10.1145\/566493.566495","article-title":"Principles of survey research part 2: designing a survey","volume":"27","author":"Kitchenham","year":"2002","journal-title":"ACM SIGSOFT Softw. Eng. Notes"},{"key":"10.1016\/j.jss.2026.112925_b34","doi-asserted-by":"crossref","unstructured":"Kong, A., Zhao, S., Chen, H., Li, Q., Qin, Y., Sun, R., Zhou, X., Wang, E., Dong, X., 2024. Better zero-shot reasoning with role-play prompting. In: Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers). pp. 4099\u20134113.","DOI":"10.18653\/v1\/2024.naacl-long.228"},{"key":"10.1016\/j.jss.2026.112925_b35","series-title":"Survey Research","author":"Krosnick","year":"2014"},{"key":"10.1016\/j.jss.2026.112925_b36","doi-asserted-by":"crossref","unstructured":"Li, S., Guo, J., Lou, J.-G., Fan, M., Liu, T., Zhang, D., 2022. Testing machine learning systems in industry: an empirical study. In: Proceedings of the 44th International Conference on Software Engineering: Software Engineering in Practice. pp. 263\u2013272.","DOI":"10.1145\/3510457.3513036"},{"key":"10.1016\/j.jss.2026.112925_b37","series-title":"10th Joint Meeting on Foundations of Software Engineering","first-page":"415","article-title":"How practitioners perceive the relevance of software engineering research","author":"Lo","year":"2015"},{"key":"10.1016\/j.jss.2026.112925_b38","series-title":"2018 IEEE 29th International Symposium on Software Reliability Engineering","first-page":"100","article-title":"Deepmutation: Mutation testing of deep learning systems","author":"Ma","year":"2018"},{"key":"10.1016\/j.jss.2026.112925_b39","doi-asserted-by":"crossref","first-page":"46534","DOI":"10.52202\/075280-2019","article-title":"Self-refine: Iterative refinement with self-feedback","volume":"36","author":"Madaan","year":"2023","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"10.1016\/j.jss.2026.112925_b40","series-title":"2021 IEEE\/ACM 1st Workshop on AI Engineering-Software Engineering for AI","first-page":"109","article-title":"Who needs MLOps: What data scientists seek to accomplish and how can mlops help?","author":"M\u00e4kinen","year":"2021"},{"key":"10.1016\/j.jss.2026.112925_b41","first-page":"77","article-title":"Matchmaker: Data drift mitigation in machine learning for large-scale systems","volume":"4","author":"Mallick","year":"2022","journal-title":"Proc. Mach. Learn. Syst."},{"key":"10.1016\/j.jss.2026.112925_b42","series-title":"2019 IEEE International Conference on Artificial Intelligence Testing (AITest)","first-page":"101","article-title":"Challenges of testing machine learning based systems","author":"Marijan","year":"2019"},{"key":"10.1016\/j.jss.2026.112925_b43","doi-asserted-by":"crossref","unstructured":"Martin-Lopez, A., 2020. AI-driven web API testing. In: Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering: Companion Proceedings. pp. 202\u2013205.","DOI":"10.1145\/3377812.3381388"},{"issue":"2","key":"10.1016\/j.jss.2026.112925_b44","doi-asserted-by":"crossref","first-page":"202","DOI":"10.1177\/1558689815607096","article-title":"The value of mixed methods research: A mixed methods study","volume":"11","author":"McKim","year":"2017","journal-title":"J. Mix. Methods Res."},{"issue":"10","key":"10.1016\/j.jss.2026.112925_b45","doi-asserted-by":"crossref","first-page":"547","DOI":"10.3928\/01484834-20130920-10","article-title":"Value of a pilot study","volume":"52","author":"Morin","year":"2013","journal-title":"J. Nurs. Educ."},{"key":"10.1016\/j.jss.2026.112925_b46","doi-asserted-by":"crossref","unstructured":"Morris, J., Lifland, E., Yoo, J.Y., Grigsby, J., Jin, D., Qi, Y., 2020. Textattack: A framework for adversarial attacks, data augmentation, and adversarial training in nlp. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations. pp. 119\u2013126.","DOI":"10.18653\/v1\/2020.emnlp-demos.16"},{"key":"10.1016\/j.jss.2026.112925_b47","unstructured":"Nemoto, T., Beglar, D., 2014. Likert-scale questionnaires. In: JALT 2013 Conference Proceedings. pp. 1\u20138."},{"issue":"8","key":"10.1016\/j.jss.2026.112925_b48","doi-asserted-by":"crossref","first-page":"2749","DOI":"10.3390\/app10082749","article-title":"A survey on theories and applications for self-driving cars based on deep learning methods","volume":"10","author":"Ni","year":"2020","journal-title":"Appl. Sci."},{"issue":"1","key":"10.1016\/j.jss.2026.112925_b49","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3680463","article-title":"An empirical study of testing machine learning in the wild","volume":"34","author":"Openja","year":"2024","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"10.1016\/j.jss.2026.112925_b50","series-title":"Software Testing and Analysis: Process, Principles, and Techniques","author":"Pezz\u00e8","year":"2008"},{"key":"10.1016\/j.jss.2026.112925_b51","doi-asserted-by":"crossref","unstructured":"Pinto, G., Miranda, B., Dissanayake, S., d\u2019Amorim, M., Treude, C., Bertolino, A., 2020. What is the vocabulary of flaky tests?. In: Proceedings of the 17th International Conference on Mining Software Repositories. pp. 492\u2013502.","DOI":"10.1145\/3379597.3387482"},{"key":"10.1016\/j.jss.2026.112925_b52","doi-asserted-by":"crossref","unstructured":"Pontillo, V., 2022. Static test flakiness prediction. In: Proceedings of the ACM\/IEEE 44th International Conference on Software Engineering: Companion Proceedings. pp. 325\u2013327.","DOI":"10.1145\/3510454.3522680"},{"issue":"2","key":"10.1016\/j.jss.2026.112925_b53","doi-asserted-by":"crossref","first-page":"55","DOI":"10.1007\/s10664-023-10436-2","article-title":"Machine learning-based test smell detection","volume":"29","author":"Pontillo","year":"2024","journal-title":"Empirical Software Engineering"},{"key":"10.1016\/j.jss.2026.112925_b54","doi-asserted-by":"crossref","first-page":"112320","DOI":"10.1016\/j.jss.2024.112320","article-title":"An empirical investigation into the capabilities of anomaly detection approaches for test smell detection","volume":"222","author":"Pontillo","year":"2025","journal-title":"Journal of Systems and Software"},{"issue":"7","key":"10.1016\/j.jss.2026.112925_b55","doi-asserted-by":"crossref","first-page":"187","DOI":"10.1007\/s10664-022-10227-1","article-title":"Static test flakiness prediction: how far can we go?","volume":"27","author":"Pontillo","year":"2022","journal-title":"Empirical Software Engineering"},{"key":"10.1016\/j.jss.2026.112925_b56","series-title":"2003 International Symposium on Empirical Software Engineering, 2003. ISESE 2003. Proceedings.","first-page":"80","article-title":"Conducting on-line surveys in software engineering","author":"Punter","year":"2003"},{"issue":"5","key":"10.1016\/j.jss.2026.112925_b57","doi-asserted-by":"crossref","first-page":"139","DOI":"10.1007\/s10664-025-10676-4","article-title":"When code smells meet ml: on the lifecycle of ml-specific code smells in ml-enabled systems","volume":"30","author":"Recupito","year":"2025","journal-title":"Empirical Software Engineering"},{"key":"10.1016\/j.jss.2026.112925_b58","series-title":"Software engineering user study recruitment on prolific: An experience report","author":"Reid","year":"2022"},{"issue":"6","key":"10.1016\/j.jss.2026.112925_b59","doi-asserted-by":"crossref","first-page":"5193","DOI":"10.1007\/s10664-020-09881-0","article-title":"Testing machine learning based systems: a systematic mapping","volume":"25","author":"Riccio","year":"2020","journal-title":"Empir. Softw. Eng."},{"issue":"2","key":"10.1016\/j.jss.2026.112925_b60","doi-asserted-by":"crossref","first-page":"802","DOI":"10.1109\/TSE.2022.3163682","article-title":"Increasing the confidence of deep neural networks by coverage analysis","volume":"49","author":"Rossolini","year":"2022","journal-title":"IEEE Trans. Softw. Eng."},{"issue":"3","key":"10.1016\/j.jss.2026.112925_b61","doi-asserted-by":"crossref","first-page":"382","DOI":"10.1093\/jssam\/smw020","article-title":"Evaluating active (opt-in) and passive (opt-out) consent bias in the transfer of federal contact data to a third-party survey agency","volume":"4","author":"Sakshaug","year":"2016","journal-title":"J. Surv. Stat. Methodol."},{"key":"10.1016\/j.jss.2026.112925_b62","article-title":"Hidden technical debt in machine learning systems","volume":"28","author":"Sculley","year":"2015","journal-title":"Adv. Neural Inf. Process. Syst."},{"issue":"3","key":"10.1016\/j.jss.2026.112925_b63","doi-asserted-by":"crossref","first-page":"46","DOI":"10.1109\/MS.2018.2875968","article-title":"Metamorphic testing: Testing the untestable","volume":"37","author":"Segura","year":"2018","journal-title":"IEEE Softw."},{"key":"10.1016\/j.jss.2026.112925_b64","doi-asserted-by":"crossref","unstructured":"Serban, A., Van der Blom, K., Hoos, H., Visser, J., 2020. Adoption and effects of software engineering best practices in machine learning. In: Proceedings of the 14th ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement. ESEM, pp. 1\u201312.","DOI":"10.1145\/3382494.3410681"},{"key":"10.1016\/j.jss.2026.112925_b65","doi-asserted-by":"crossref","unstructured":"Song, Q., Borg, M., Engstr\u00f6m, E., Ard\u00f6, H., Rico, S., 2022. Exploring ML testing in practice: Lessons learned from an interactive rapid review with axis communications. In: Proceedings of the 1st International Conference on AI Engineering: Software Engineering for AI. pp. 10\u201321.","DOI":"10.1145\/3522664.3528596"},{"issue":"10","key":"10.1016\/j.jss.2026.112925_b66","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3657283","article-title":"Adapting neural networks at runtime: Current trends in at-runtime optimizations for deep learning","volume":"56","author":"Sponner","year":"2024","journal-title":"ACM Comput. Surv."},{"issue":"1","key":"10.1016\/j.jss.2026.112925_b67","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3533314","article-title":"Feedback-directed metamorphic testing","volume":"32","author":"Sun","year":"2023","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"10.1016\/j.jss.2026.112925_b68","article-title":"Adversarial AI testcases for maritime autonomous systems","author":"Tam","year":"2023","journal-title":"AI, Comput. Sci. Robot. Technol."},{"issue":"6","key":"10.1016\/j.jss.2026.112925_b69","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3665926","article-title":"AI robustness: a human-centered perspective on technological challenges and opportunities","volume":"57","author":"Tocchetti","year":"2025","journal-title":"ACM Comput. Surv."},{"key":"10.1016\/j.jss.2026.112925_b70","doi-asserted-by":"crossref","unstructured":"Wan, C., Liu, S., Xie, S., Liu, Y., Hoffmann, H., Maire, M., Lu, S., 2022. Automated testing of software that uses machine learning apis. In: Proceedings of the 44th International Conference on Software Engineering. pp. 212\u2013224.","DOI":"10.1145\/3510003.3510068"},{"key":"10.1016\/j.jss.2026.112925_b71","doi-asserted-by":"crossref","first-page":"61","DOI":"10.1016\/j.patrec.2020.07.042","article-title":"Comparative analysis of image classification algorithms based on traditional machine learning and deep learning","volume":"141","author":"Wang","year":"2021","journal-title":"Pattern Recognit. Lett."},{"issue":"10","key":"10.1016\/j.jss.2026.112925_b72","doi-asserted-by":"crossref","first-page":"209","DOI":"10.3390\/computers12100209","article-title":"On the robustness of ML-based network intrusion detection systems: An adversarial and distribution shift perspective","volume":"12","author":"Wang","year":"2023","journal-title":"Computers"},{"key":"10.1016\/j.jss.2026.112925_b73","series-title":"2023 IEEE\/ACM 20th International Conference on Mining Software Repositories","first-page":"62","article-title":"Niche: A curated dataset of engineered machine learning projects in python","author":"Widyasari","year":"2023"},{"key":"10.1016\/j.jss.2026.112925_b74","series-title":"Experimentation in Software Engineering","volume":"vol. 236","author":"Wohlin","year":"2012"},{"key":"10.1016\/j.jss.2026.112925_b75","series-title":"2022 IEEE International Conference on Software Analysis, Evolution and Reengineering","first-page":"408","article-title":"Revisiting neuron coverage metrics and quality of deep neural networks","author":"Yang","year":"2022"},{"issue":"1","key":"10.1016\/j.jss.2026.112925_b76","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1109\/TSE.2019.2962027","article-title":"Machine learning testing: Survey, landscapes and horizons","volume":"48","author":"Zhang","year":"2020","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112925_b77","series-title":"2025 IEEE\/ACM 1st International Workshop on Software Engineering for Autonomous Driving Systems (SE4ADS)","first-page":"1","article-title":"AI-augmented metamorphic testing for comprehensive validation of autonomous vehicles","author":"Zhang","year":"2025"},{"key":"10.1016\/j.jss.2026.112925_b78","series-title":"Human and Machine Learning","author":"Zhou","year":"2018"}],"container-title":["Journal of Systems and Software"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0164121226001585?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0164121226001585?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,7,4]],"date-time":"2026-07-04T08:07:52Z","timestamp":1783152472000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0164121226001585"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,10]]},"references-count":78,"alternative-id":["S0164121226001585"],"URL":"https:\/\/doi.org\/10.1016\/j.jss.2026.112925","relation":{},"ISSN":["0164-1212"],"issn-type":[{"value":"0164-1212","type":"print"}],"subject":[],"published":{"date-parts":[[2026,10]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Understanding Machine Learning testing in practice","name":"articletitle","label":"Article Title"},{"value":"Journal of Systems and Software","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.jss.2026.112925","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 The Authors. Published by Elsevier Inc.","name":"copyright","label":"Copyright"}],"article-number":"112925"}}