{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,20]],"date-time":"2026-05-20T17:47:04Z","timestamp":1779299224957,"version":"3.51.4"},"reference-count":167,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2024,10,30]],"date-time":"2024-10-30T00:00:00Z","timestamp":1730246400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"},{"start":{"date-parts":[[2024,10,30]],"date-time":"2024-10-30T00:00:00Z","timestamp":1730246400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"}],"funder":[{"name":"Ministry of Science, Technological Development and Innovation of the Republic of Serbia","award":["451-03-65\/2024-03\/200103"],"award-info":[{"award-number":["451-03-65\/2024-03\/200103"]}]},{"name":"Ministry of Science, Technological Development and Innovation of the Republic of Serbia","award":["451-03-65\/2024-03\/200103"],"award-info":[{"award-number":["451-03-65\/2024-03\/200103"]}]},{"name":"Ministry of Science, Technological Development and Innovation of the Republic of Serbia","award":["451-03-65\/2024-03\/200103"],"award-info":[{"award-number":["451-03-65\/2024-03\/200103"]}]},{"DOI":"10.13039\/501100016047","name":"Science Fund of the Republic of Serbia","doi-asserted-by":"publisher","award":["11113"],"award-info":[{"award-number":["11113"]}],"id":[{"id":"10.13039\/501100016047","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J Big Data"],"DOI":"10.1186\/s40537-024-01028-y","type":"journal-article","created":{"date-parts":[[2024,10,30]],"date-time":"2024-10-30T16:02:41Z","timestamp":1730304161000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":27,"title":["Quality assurance strategies for machine learning applications in big data analytics: an overview"],"prefix":"10.1186","volume":"11","author":[{"given":"Mihajlo","family":"Ogrizovi\u0107","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dra\u017een","family":"Dra\u0161kovi\u0107","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dragan","family":"Boji\u0107","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,10,30]]},"reference":[{"key":"1028_CR1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-70008-8","volume-title":"Theory and practice of quality assurance for machine learning systems","author":"S Ackerman","year":"2024","unstructured":"Ackerman S, Barash G, Farchi E, Raz O, Shehory O. Theory and practice of quality assurance for machine learning systems. New York: Springer; 2024."},{"key":"1028_CR2","first-page":"1","volume":"3","author":"B Alhijawi","year":"2023","unstructured":"Alhijawi B, Awajan A. Genetic algorithms: theory, genetic operators, solutions, and applications. Evol Intel. 2023;3:1\u20132.","journal-title":"Evol Intel"},{"issue":"72","key":"1028_CR3","doi-asserted-by":"publisher","first-page":"327","DOI":"10.1016\/j.eswa.2016.10.055","volume":"15","author":"G Amato","year":"2017","unstructured":"Amato G, Carrara F, Falchi F, Gennaro C, Meghini C, Vairo C. Deep learning for decentralized parking lot occupancy detection. Expert Syst Appl. 2017;15(72):327\u201334.","journal-title":"Expert Syst Appl"},{"issue":"4","key":"1028_CR4","doi-asserted-by":"publisher","first-page":"342","DOI":"10.1109\/TG.2021.3075065","volume":"13","author":"E Antonioni","year":"2021","unstructured":"Antonioni E, Suriani V, Riccio F, Nardi D. Game strategies for physical robot soccer players: a survey. IEEE Trans Games. 2021;13(4):342\u201357.","journal-title":"IEEE Trans Games"},{"key":"1028_CR5","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10916-018-1088-1","volume":"42","author":"SM Anwar","year":"2018","unstructured":"Anwar SM, Majid M, Qayyum A, Awais M, Alnowami M, Khan MK. Medical image analysis using convolutional neural networks: a review. J Med Syst. 2018;42:1\u20133.","journal-title":"J Med Syst"},{"issue":"2","key":"1028_CR6","doi-asserted-by":"publisher","first-page":"2739","DOI":"10.32604\/cmc.2023.030266","volume":"74","author":"I Arshad","year":"2023","unstructured":"Arshad I, Alsamhi SH, Afzal W. Big data testing techniques: taxonomy, challenges and future trends. Comput, Mater Continua. 2023;74(2):2739\u201370.","journal-title":"Comput, Mater Continua"},{"issue":"1","key":"1028_CR7","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1186\/s40537-021-00445-7","volume":"8","author":"FA Batarseh","year":"2021","unstructured":"Batarseh FA, Freeman L, Huang CH. A survey on artificial intelligence assurance. J Big Data. 2021;8(1):60.","journal-title":"J Big Data"},{"issue":"11","key":"1028_CR8","doi-asserted-by":"publisher","first-page":"e0242050","DOI":"10.1371\/journal.pone.0242050","volume":"15","author":"V Batanovi\u0107","year":"2020","unstructured":"Batanovi\u0107 V, Cvetanovi\u0107 M, Nikoli\u0107 B. A versatile framework for resource-limited sentiment articulation, annotation, and analysis of short texts. PLoS ONE. 2020;15(11):e0242050.","journal-title":"PLoS ONE"},{"issue":"52","key":"1028_CR9","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10462-018-9654-y","volume":"1","author":"Z Batmaz","year":"2019","unstructured":"Batmaz Z, Yurekli A, Bilge A, Kaleli C. A review on deep learning for recommender systems: challenges and remedies. Artif Intell Rev. 2019;1(52):1\u201337.","journal-title":"Artif Intell Rev"},{"issue":"1","key":"1028_CR10","doi-asserted-by":"publisher","first-page":"11","DOI":"10.1145\/332148.332153","volume":"3","author":"J Beck","year":"1996","unstructured":"Beck J, Stern M, Haugsjaa E. Applications of AI in education. XRDS: Crossroads, ACM Mag Stud. 1996;3(1):11\u20135.","journal-title":"XRDS: Crossroads, ACM Mag Stud"},{"key":"1028_CR11","first-page":"1","volume-title":"A systematic approach to generation of new ideas for PhD research in computing","author":"V Blagojevi\u0107","year":"2017","unstructured":"Blagojevi\u0107 V, Boji\u0107 D, Bojovi\u0107 M, Cvetanovi\u0107 M, \u0110or\u0111evi\u0107 J, \u0110ur\u0111evi\u0107 \u0110, Furlan B, Gajin S, Jovanovi\u0107 Z, Mili\u0107ev D, Milutinovi\u0107 V. A systematic approach to generation of new ideas for PhD research in computing. Amsterdam: Elsevier; 2017. p. 1\u201331."},{"issue":"213","key":"1028_CR12","doi-asserted-by":"publisher","first-page":"118912","DOI":"10.1016\/j.eswa.2022.118912","volume":"1","author":"S Boeschoten","year":"2023","unstructured":"Boeschoten S, Catal C, Tekinerdogan B, Lommen A, Blokland M. The automation of the development of classification models and improvement of model quality using feature engineering techniques. Expert Syst Appl. 2023;1(213):118912.","journal-title":"Expert Syst Appl"},{"issue":"164","key":"1028_CR13","doi-asserted-by":"publisher","first-page":"110542","DOI":"10.1016\/j.jss.2020.110542","volume":"1","author":"HB Braiek","year":"2020","unstructured":"Braiek HB, Khomh F. On testing machine learning programs. J Syst Softw. 2020;1(164):110542.","journal-title":"J Syst Softw"},{"key":"1028_CR14","first-page":"1123","volume-title":"The ML test score: A rubric for ML production readiness and technical debt reduction","author":"E Breck","year":"2017","unstructured":"Breck E, Cai S, Nielsen E, Salib M, Sculley D. The ML test score: A rubric for ML production readiness and technical debt reduction. Piscataway: IEEE; 2017. p. 1123\u201332."},{"key":"1028_CR15","doi-asserted-by":"publisher","DOI":"10.4855\/arXiv.2207.14529","author":"L Budach","year":"2022","unstructured":"Budach L, Feuerpfeil M, Ihde N, Nathansen A, Noack N, Patzlaff H, Naumann F, Harmouch H. The effects of data quality on machine learning performance. arXiv preprint. 2022. https:\/\/doi.org\/10.4855\/arXiv.2207.14529.","journal-title":"arXiv preprint"},{"key":"1028_CR16","first-page":"63","volume-title":"Input prioritization for testing neural networks","author":"T Byun","year":"2019","unstructured":"Byun T, Sharma V, Vijayakumar A, Rayadurgam S, Cofer D. Input prioritization for testing neural networks. Piscataway: IEEE; 2019. p. 63\u201370."},{"key":"1028_CR17","first-page":"39","volume-title":"Towards evaluating the robustness of neural networks","author":"N Carlini","year":"2017","unstructured":"Carlini N, Wagner D. Towards evaluating the robustness of neural networks. Piscataway: IEEE; 2017. p. 39\u201357."},{"key":"1028_CR18","first-page":"41","volume-title":"Preproduction deploys: cloud-native integration testing","author":"JJ Carroll","year":"2021","unstructured":"Carroll JJ, Anand P, Guo D. Preproduction deploys: cloud-native integration testing. Piscataway: IEEE; 2021. p. 41\u20138."},{"key":"1028_CR19","doi-asserted-by":"publisher","DOI":"10.4855\/arXiv.2002.12543","author":"TY Chen","year":"2020","unstructured":"Chen TY, Cheung SC, Yiu SM. Metamorphic testing: a new approach for generating next test cases. arXiv preprint. 2020. https:\/\/doi.org\/10.4855\/arXiv.2002.12543.","journal-title":"arXiv preprint"},{"issue":"2","key":"1028_CR20","doi-asserted-by":"publisher","first-page":"831","DOI":"10.1109\/TR.2021.3070863","volume":"70","author":"H Chen","year":"2021","unstructured":"Chen H, Chen J, Ding J. Data evaluation and enhancement for quality improvement of machine learning. IEEE Trans Reliab. 2021;70(2):831\u201347.","journal-title":"IEEE Trans Reliab"},{"key":"1028_CR21","first-page":"210","volume-title":"Big data system testing method based on chaos engineering","author":"G Chen","year":"2022","unstructured":"Chen G, Bai G, Zhang C, Wang J, Ni K, Chen Z. Big data system testing method based on chaos engineering. Piscataway: IEEE; 2022. p. 210\u20135."},{"issue":"5","key":"1028_CR22","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3652155","volume":"33","author":"Z Chen","year":"2024","unstructured":"Chen Z, Zhang JM, Hort M, Harman M, Sarro F. Fairness testing: a comprehensive survey and analysis of trends. ACM Trans Softw Eng Methodol. 2024;33(5):1\u201359.","journal-title":"ACM Trans Softw Eng Methodol"},{"key":"1028_CR23","first-page":"1","volume-title":"Towards dependability metrics for neural networks","author":"CH Cheng","year":"2018","unstructured":"Cheng CH, Huang CH, Ruess H, Yasuoka H. Towards dependability metrics for neural networks. Piscataway: IEEE; 2018. p. 1\u20134."},{"key":"1028_CR24","doi-asserted-by":"publisher","DOI":"10.1109\/MAHC.2023.3300660","author":"BJ Copeland","year":"2023","unstructured":"Copeland BJ. Early AI in Britain: Turing et al. IEEE Ann Hist Comput. 2023. https:\/\/doi.org\/10.1109\/MAHC.2023.3300660.","journal-title":"IEEE Ann Hist Comput"},{"key":"1028_CR25","doi-asserted-by":"publisher","DOI":"10.1016\/j.patter.2023.100857","author":"NK Corr\u00eaa","year":"2023","unstructured":"Corr\u00eaa NK, Galv\u00e3o C, Santos JW, Del Pino C, Pinto EP, Barbosa C, Massmann D, Mambrini R, Galv\u00e3o L, Terem E, de Oliveira N. Worldwide AI ethics: a review of 200 guidelines and recommendations for AI governance. Patterns. 2023. https:\/\/doi.org\/10.1016\/j.patter.2023.100857.","journal-title":"Patterns"},{"key":"1028_CR26","doi-asserted-by":"crossref","unstructured":"Dai W, Yoshigoe K, Parsley W. Improving data quality through deep learning and statistical models. In: Information Technology-New Generations: 14th International Conference on Information Technology 2018. Springer International Publishing. pp. 515-522.","DOI":"10.1007\/978-3-319-54978-1_66"},{"key":"1028_CR27","doi-asserted-by":"publisher","DOI":"10.4855\/arXiv.2207.07528","author":"G d'Aloisio","year":"2022","unstructured":"d\u2019Aloisio G, Di Marco A, Stilo G. Modeling quality and machine learning pipelines through extended feature models. arXiv preprint. 2022. https:\/\/doi.org\/10.4855\/arXiv.2207.07528.","journal-title":"arXiv preprint"},{"issue":"4","key":"1028_CR28","doi-asserted-by":"publisher","first-page":"2709","DOI":"10.1007\/s10462-019-09744-1","volume":"53","author":"A Da\u2019u","year":"2020","unstructured":"Da\u2019u A, Salim N. Recommendation system based on deep learning methods: a systematic review and new directions. Artif Intell Rev. 2020;53(4):2709\u201348.","journal-title":"Artif Intell Rev"},{"key":"1028_CR29","doi-asserted-by":"publisher","DOI":"10.1155\/2019\/4368036","author":"V Deli\u0107","year":"2019","unstructured":"Deli\u0107 V, Peri\u0107 Z, Se\u010dujski M, Jakovljevi\u0107 N, Nikoli\u0107 J, Mi\u0161kovi\u0107 D, Simi\u0107 N, Suzi\u0107 S, Deli\u0107 T. Speech technology progress based on new machine learning paradigm. Comput Intell Neurosci. 2019. https:\/\/doi.org\/10.1155\/2019\/4368036.","journal-title":"Comput Intell Neurosci"},{"issue":"135","key":"1028_CR30","doi-asserted-by":"publisher","first-page":"108517","DOI":"10.1016\/j.ecolind.2021.108517","volume":"1","author":"FA Diaz-Gonzalez","year":"2022","unstructured":"Diaz-Gonzalez FA, Vuelvas J, Correa CA, Vallejo VE, Patino D. Machine learning and remote sensing techniques applied to estimate soil indicators\u2013review. Ecol Ind. 2022;1(135):108517.","journal-title":"Ecol Ind"},{"key":"1028_CR31","doi-asserted-by":"crossref","unstructured":"Ding Z, Wang Y, Wang G, Zhang D, Kifer D. Detecting violations of differential privacy. In: Proceedings of the 2018 ACM SIGSAC Conference on Computer and Communications Security 2018 Oct 15. pp. 475\u2013489.","DOI":"10.1145\/3243734.3243818"},{"issue":"5","key":"1028_CR32","doi-asserted-by":"publisher","first-page":"1195","DOI":"10.1002\/cae.21988","volume":"26","author":"D Draskovic","year":"2018","unstructured":"Draskovic D, Cvetanovic M, Nikolic B. SAIL\u2014Software system for learning AI algorithms. Comput Appl Eng Educ. 2018;26(5):1195\u2013216.","journal-title":"Comput Appl Eng Educ"},{"issue":"18","key":"1028_CR33","doi-asserted-by":"publisher","first-page":"3236","DOI":"10.3390\/math10183236","volume":"10","author":"D Draskovic","year":"2022","unstructured":"Draskovic D, Zecevic D, Nikolic B. Development of a multilingual model for machine sentiment analysis in the Serbian language. Mathematics. 2022;10(18):3236.","journal-title":"Mathematics"},{"key":"1028_CR34","doi-asserted-by":"crossref","unstructured":"Dutta S, Zhang W, Huang Z, Misailovic S. Storm: program reduction for testing and debugging probabilistic programming systems. In: Proceedings of the 2019 27th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering 2019 Aug 12. pp. 729\u2013739.","DOI":"10.1145\/3338906.3338972"},{"key":"1028_CR35","doi-asserted-by":"crossref","unstructured":"Dwarakanath A, Ahuja M, Sikand S, Rao RM, Bose RJ, Dubash N, Podder S. Identifying implementation bugs in machine learning based image classifiers using metamorphic testing. In:  Proceedings of the 27th ACM SIGSOFT international symposium on software testing and analysis 2018 Jul 12. pp. 118\u2013128.","DOI":"10.1145\/3213846.3213858"},{"issue":"6650","key":"1028_CR36","doi-asserted-by":"publisher","first-page":"1110","DOI":"10.1126\/science.adh4451","volume":"380","author":"Z Epstein","year":"2023","unstructured":"Epstein Z, Hertzmann A, Akten M, Farid H, Fjeld J, Frank MR, Groh M, Herman L, Leach N, Mahari R. Art and the science of generative AI. Science. 2023;380(6650):1110\u20131.","journal-title":"Science"},{"issue":"12","key":"1028_CR37","first-page":"e2113","volume":"30","author":"A Fabijan","year":"2018","unstructured":"Fabijan A, Dmitriev P, McFarland C, Vermeer L, Holmstr\u00f6m Olsson H, Bosch J. Experimentation growth: evolving trustworthy A\/B testing capabilities in online software companies. J Softw: Evol Process. 2018;30(12):e2113.","journal-title":"J Softw: Evol Process"},{"issue":"3","key":"1028_CR38","first-page":"1","volume":"15","author":"H Fadlallah","year":"2023","unstructured":"Fadlallah H, Kilany R, Dhayne H, El Haddad R, Haque R, Taher Y, Jaber A. Context-aware big data quality assessment: a scoping review. ACM J Data Inform Quality. 2023;15(3):1\u201333.","journal-title":"ACM J Data Inform Quality"},{"issue":"1","key":"1028_CR39","doi-asserted-by":"publisher","first-page":"111","DOI":"10.1007\/s12599-023-00834-7","volume":"66","author":"S Feuerriegel","year":"2024","unstructured":"Feuerriegel S, Hartmann J, Janiesch C, Zschech P. Generative ai. Bus Inf Syst Eng. 2024;66(1):111\u201326.","journal-title":"Bus Inf Syst Eng"},{"issue":"3","key":"1028_CR40","first-page":"277","volume":"25","author":"F Fui-Hoon Nah","year":"2023","unstructured":"Fui-Hoon Nah F, Zheng R, Cai J, Siau K, Chen L. Generative AI and ChatGPT: applications, challenges, and AI-human collaboration. J Inform Technol Case Appl Res. 2023;25(3):277\u2013304.","journal-title":"J Inform Technol Case Appl Res"},{"key":"1028_CR41","doi-asserted-by":"crossref","unstructured":"Galhotra S, Brun Y, Meliou A. Fairness testing: testing software for discrimination. In: Proceedings of the 2017 11th Joint meeting on foundations of software engineering 2017 Aug 21. pp. 498\u2013510.","DOI":"10.1145\/3106237.3106277"},{"key":"1028_CR42","first-page":"1","volume-title":"Analysis of the complexity of the automatic pipeline generation problem","author":"U Garciarena","year":"2018","unstructured":"Garciarena U, Santana R, Mendiburu A. Analysis of the complexity of the automatic pipeline generation problem. Piscataway: IEEE; 2018. p. 1\u20138."},{"issue":"1","key":"1028_CR43","doi-asserted-by":"publisher","first-page":"113","DOI":"10.18421\/TEM81-16","volume":"8","author":"E Gegic","year":"2019","unstructured":"Gegic E, Isakovic B, Keco D, Masetic Z, Kevric J. Car price prediction using machine learning techniques. TEM J. 2019;8(1):113.","journal-title":"TEM J"},{"key":"1028_CR44","unstructured":"Goodfellow I, Pouget-Abadie J, Mirza M, Xu B, Warde-Farley D, Ozair S, Courville A, Bengio Y. Generative adversarial nets. Adv Neural Infor Process Syst. 2014;27."},{"key":"1028_CR45","doi-asserted-by":"publisher","DOI":"10.4855\/arXiv.1412.5218","author":"RB Grosse","year":"2014","unstructured":"Grosse RB, Duvenaud DK. Testing mcmc code. arXiv preprint. 2014. https:\/\/doi.org\/10.4855\/arXiv.1412.5218.","journal-title":"arXiv preprint"},{"issue":"1","key":"1028_CR46","first-page":"65","volume":"11","author":"M Gudipati","year":"2013","unstructured":"Gudipati M, Rao S, Mohan ND, Gajja NK. Big data: testing approach to overcome quality challenges. Big Data: Chall Oppor. 2013;11(1):65\u201372.","journal-title":"Big Data: Chall Oppor"},{"key":"1028_CR47","doi-asserted-by":"crossref","unstructured":"Guo J, Jiang Y, Zhao Y, Chen Q, Sun J. Dlfuzz: Differential fuzzing testing of deep learning systems. In: Proceedings of the 2018 26th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering 2018 Oct 26. pp. 739\u2013743.","DOI":"10.1145\/3236024.3264835"},{"key":"1028_CR48","volume-title":"A zero-trust methodology for security of complex systems with machine learning components","author":"B Hale","year":"2017","unstructured":"Hale B, Van Bossuyt DL, Papakonstantinou N, O\u2019Halloran B. A zero-trust methodology for security of complex systems with machine learning components. New York City: American Society of Mechanical Engineers; 2017."},{"issue":"5","key":"1028_CR49","doi-asserted-by":"publisher","first-page":"3201","DOI":"10.1007\/s10462-019-09760-1","volume":"53","author":"AE Hassanien","year":"2020","unstructured":"Hassanien AE, Darwish A, Abdelghafar S. Machine learning in telemetry data mining of space mission: basics, challenging and future directions. Artif Intell Rev. 2020;53(5):3201\u201330.","journal-title":"Artif Intell Rev"},{"key":"1028_CR50","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4842-7032-5_8","author":"H Helskyaho","year":"2021","unstructured":"Helskyaho H, Yu J, Yu K, Helskyaho H, Yu J, Yu K. Delivery and automation pipeline in machine learning. Mach Learn Oracle Database Prof: Deploy Model-Driven Appl Autom Pipelines. 2021. https:\/\/doi.org\/10.1007\/978-1-4842-7032-5_8.","journal-title":"Mach Learn Oracle Database Prof: Deploy Model-Driven Appl Autom Pipelines"},{"key":"1028_CR51","first-page":"151","volume-title":"Applying machine learning with chaos engineering","author":"J Hern\u00e1ndez-Serrato","year":"2020","unstructured":"Hern\u00e1ndez-Serrato J, Velasco A, Nifio Y, Linares-V\u00e1squez M. Applying machine learning with chaos engineering. Piscataway: IEEE; 2020. p. 151\u20132."},{"issue":"3","key":"1028_CR52","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1007\/s13347-022-00542-2","volume":"35","author":"J Himmelreich","year":"2022","unstructured":"Himmelreich J, K\u00f6hler S. Responsible AI through conceptual engineering. Philos Technol. 2022;35(3):60.","journal-title":"Philos Technol"},{"key":"1028_CR53","volume-title":"Behind deep blue: building the computer that defeated the world chess champion","author":"FH Hsu","year":"2002","unstructured":"Hsu FH. Behind deep blue: building the computer that defeated the world chess champion. Princeton: Princeton University Press; 2002."},{"key":"1028_CR54","first-page":"362","volume-title":"A collection of software engineering challenges for big data system development","author":"O Hummel","year":"2018","unstructured":"Hummel O, Eichelberger H, Giloj A, Werle D, Schmid K. A collection of software engineering challenges for big data system development. Piscataway: IEEE; 2018. p. 362\u20139."},{"key":"1028_CR55","unstructured":"Hynes N, Sculley D, Terry M. The data linter: Lightweight, automated sanity checking for ml data sets. InNIPS MLSys Workshop 2017 (Vol. 1, No. 5)."},{"key":"1028_CR56","unstructured":"ISO IEC 9126:2001. Software engineering\u2014product quality. International organization for standardization, international electrotechnical commission: Tech. rep. 2001."},{"key":"1028_CR57","unstructured":"ISO IEC 25000:2014. Systems and software engineering\u2014Systems and software Quality Requirements and Evaluation (SQuaRE) - Guide to SQuaRE. International organization for standardization, international electrotechnical commission: Standard. 2014."},{"key":"1028_CR58","doi-asserted-by":"crossref","unstructured":"Jernberg H, Runeson P, Engstr\u00f6m E. Getting Started with Chaos Engineering-design of an implementation framework in practice. In: Proceedings of the 14th ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM) 2020 Oct 5. pp. 1\u201310.","DOI":"10.1145\/3382494.3421464"},{"issue":"1","key":"1028_CR59","doi-asserted-by":"publisher","first-page":"214","DOI":"10.1109\/TCSS.2020.3021467","volume":"8","author":"S Ji","year":"2020","unstructured":"Ji S, Pan S, Li X, Cambria E, Long G, Huang Z. Suicidal ideation detection: a review of machine learning methods and applications. IEEE Trans Comput Soc Syst. 2020;8(1):214\u201326.","journal-title":"IEEE Trans Comput Soc Syst"},{"issue":"22","key":"1028_CR60","doi-asserted-by":"publisher","first-page":"8052","DOI":"10.3390\/app10228052","volume":"10","author":"S Ji","year":"2020","unstructured":"Ji S, Li Q, Cao W, Zhang P, Muccini H. Quality assurance technologies of big data applications: a systematic literature review. Appl Sci. 2020;10(22):8052.","journal-title":"Appl Sci"},{"issue":"1","key":"1028_CR61","doi-asserted-by":"publisher","first-page":"2031","DOI":"10.1007\/s11042-023-15767-2","volume":"83","author":"V Jocovic","year":"2024","unstructured":"Jocovic V, Marinkovic M, Stojanovic S, Nikolic B. Automated assessment of pen and paper tests using computer vision. Multimed Tools Appl. 2024;83(1):2031\u201352.","journal-title":"Multimed Tools Appl"},{"issue":"8","key":"1028_CR62","doi-asserted-by":"publisher","first-page":"11765","DOI":"10.1007\/s11042-020-10183-2","volume":"80","author":"RK Kaliyar","year":"2021","unstructured":"Kaliyar RK, Goswami A, Narang P. FakeBERT: fake news detection in social media with a BERT-based deep learning approach. Multimed Tools Appl. 2021;80(8):11765\u201388.","journal-title":"Multimed Tools Appl"},{"issue":"2","key":"1028_CR63","first-page":"481","volume":"15","author":"D Kang","year":"2020","unstructured":"Kang D, Raghavan D, Bailis P, Zaharia M. Model assertions for monitoring and improving ML models. Proc Mach Learn Syst. 2020;15(2):481\u201396.","journal-title":"Proc Mach Learn Syst"},{"key":"1028_CR64","unstructured":"Karpathy, A. CS231n Convolutional Neural Networks for Visual Recognition. http:\/\/cs231n.github.io\/neural-networks-3\/. 2018."},{"key":"1028_CR65","doi-asserted-by":"crossref","unstructured":"K\u00e4stner C, Kang E. Teaching software engineering for AI-enabled systems. In: Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering: Software Engineering Education and Training 2020 Jun 27. pp. 45\u201348.","DOI":"10.1145\/3377814.3381714"},{"key":"1028_CR66","unstructured":"K\u00e4stner, C. Data quality for building production ML systems. Medium. https:\/\/ckaestne.medium.com\/data-quality-for-building-production-ml-systems-2e0cc7e6113f. 2021."},{"key":"1028_CR67","unstructured":"K\u00e4stner, C. Machine learning testing and experimenting in production. Medium. https:\/\/ckaestne.medium.com\/quality-assurance-in-production-for-ml-enabled-systems-4d1b3442316f. 2021."},{"key":"1028_CR68","unstructured":"K\u00e4stner, C. Versioning, provenance, and reproducibility in production machine learning. Medium. https:\/\/ckaestne.medium.com\/versioning-provenance-and-reproducibility-in-production-machine-learning-355c48665005. 2021."},{"key":"1028_CR69","unstructured":"K\u00e4stner, C. Quality assurance for machine-learning pipelines. Medium. https:\/\/ckaestne.medium.com\/quality-assurance-for-machine-learning-pipelines-d495b8e5ad6a. 2022."},{"key":"1028_CR70","unstructured":"K\u00e4stner, C. System quality. Medium. https:\/\/ckaestne.medium.com\/integration-and-system-testing-bc4db6650d1. 2022."},{"key":"1028_CR71","unstructured":"K\u00e4stner, C. Transparency and accountability in ML-enabled systems. Medium. https:\/\/ckaestne.medium.com\/transparency-and-accountability-in-ml-enabled-systems-f8ed0b6fd183. 2022."},{"key":"1028_CR72","unstructured":"K\u00e4stner, C. Model quality. Medium. https:\/\/ckaestne.medium.com\/model-quality-d654cea6944b. 2024."},{"issue":"4","key":"1028_CR73","doi-asserted-by":"publisher","first-page":"807","DOI":"10.1016\/j.gie.2020.06.040","volume":"92","author":"V Kaul","year":"2020","unstructured":"Kaul V, Enslin S, Gross SA. History of artificial intelligence in medicine. Gastrointest Endosc. 2020;92(4):807\u201312.","journal-title":"Gastrointest Endosc"},{"issue":"15","key":"1028_CR74","doi-asserted-by":"publisher","first-page":"104","DOI":"10.1016\/j.csbj.2016.12.005","volume":"1","author":"I Kavakiotis","year":"2017","unstructured":"Kavakiotis I, Tsave O, Salifoglou A, Maglaveras N, Vlahavas I, Chouvarda I. Machine learning and data mining methods in diabetes research. Comput Struct Biotechnol J. 2017;1(15):104\u201316.","journal-title":"Comput Struct Biotechnol J"},{"issue":"200","key":"1028_CR75","doi-asserted-by":"publisher","first-page":"1288","DOI":"10.1016\/j.procs.2022.01.330","volume":"1","author":"A Kharitonov","year":"2022","unstructured":"Kharitonov A, Nahhas A, Pohl M, Turowski K. Comparative analysis of machine learning models for anomaly detection in manufacturing. Proc Comput Sci. 2022;1(200):1288\u201397.","journal-title":"Proc Comput Sci"},{"key":"1028_CR76","first-page":"1039","volume-title":"Guiding deep learning system testing using surprise adequacy","author":"J Kim","year":"2019","unstructured":"Kim J, Feldt R, Yoo S. Guiding deep learning system testing using surprise adequacy. Piscataway: IEEE; 2019. p. 1039\u201349."},{"key":"1028_CR77","volume-title":"Thoughtful machine learning: a test-driven approach","author":"M Kirk","year":"2014","unstructured":"Kirk M. Thoughtful machine learning: a test-driven approach. Sebastopol: O\u2019Reilly Media, Inc.; 2014."},{"key":"1028_CR78","doi-asserted-by":"publisher","DOI":"10.1093\/jamia\/ocae059","author":"T Knez","year":"2024","unstructured":"Knez T, \u017ditnik S. Multimodal learning for temporal relation extraction in clinical texts. J Am Med Inform Assoc. 2024. https:\/\/doi.org\/10.1093\/jamia\/ocae059.","journal-title":"J Am Med Inform Assoc"},{"key":"1028_CR79","doi-asserted-by":"publisher","first-page":"140","DOI":"10.1007\/s10618-008-0114-1","volume":"18","author":"R Kohavi","year":"2009","unstructured":"Kohavi R, Longbotham R, Sommerfield D, Henne RM. Controlled experiments on the web: survey and practical guide. Data Min Knowl Disc. 2009;18:140\u201381.","journal-title":"Data Min Knowl Disc"},{"issue":"3","key":"1028_CR80","first-page":"59","volume":"38","author":"S Krishnan","year":"2015","unstructured":"Krishnan S, Wang J, Franklin MJ, Goldberg K, Kraska T, Milo T, Wu E. SampleClean: fast and reliable analytics on dirty data. IEEE Data Eng Bull. 2015;38(3):59\u201375.","journal-title":"IEEE Data Eng Bull"},{"issue":"2","key":"1028_CR81","doi-asserted-by":"publisher","first-page":"712","DOI":"10.1109\/TITS.2019.2962338","volume":"22","author":"S Kuutti","year":"2020","unstructured":"Kuutti S, Bowden R, Jin Y, Barber P, Fallah S. A survey of deep learning applications to autonomous vehicle control. IEEE Trans Intell Transp Syst. 2020;22(2):712\u201333.","journal-title":"IEEE Trans Intell Transp Syst"},{"issue":"5","key":"1028_CR82","doi-asserted-by":"publisher","first-page":"1103","DOI":"10.1007\/s10994-020-05872-w","volume":"109","author":"H Kuwajima","year":"2020","unstructured":"Kuwajima H, Yasuoka H, Nakae T. Engineering problems in machine learning systems. Mach Learn. 2020;109(5):1103\u201326.","journal-title":"Mach Learn"},{"key":"1028_CR83","doi-asserted-by":"crossref","unstructured":"Laato S, Birkstedt T, M\u00e4antym\u00e4ki M, Minkkinen M, Mikkonen T. AI governance in the system development life cycle: Insights on responsible machine learning engineering. In: Proceedings of the 1st International Conference on AI Engineering: Software Engineering for AI 2022 May 16. pp. 113\u2013123.","DOI":"10.1145\/3522664.3528598"},{"key":"1028_CR84","doi-asserted-by":"crossref","unstructured":"Li N, Lei Y, Khan HR, Liu J, Guo Y. Applying combinatorial test data generation to big data applications. In: Proceedings of the 31st IEEE\/ACM International Conference on Automated Software Engineering 2016 Aug 25. pp. 637\u2013647.","DOI":"10.1145\/2970276.2970325"},{"key":"1028_CR85","first-page":"187","volume-title":"Issues with conducting controlled on-line experiments for e-commerce","author":"D Liu","year":"2017","unstructured":"Liu D, Xu S, Zhang B, Wang C, Li C, Zhou F. Issues with conducting controlled on-line experiments for e-commerce. Piscataway: IEEE; 2017. p. 187\u201392."},{"issue":"5","key":"1028_CR86","doi-asserted-by":"publisher","first-page":"1178","DOI":"10.1111\/1540-6229.12397","volume":"51","author":"F Lorenz","year":"2023","unstructured":"Lorenz F, Willwersch J, Cajias M, Fuerst F. Interpretable machine learning for real estate market analysis. Real Estate Econ. 2023;51(5):1178\u2013208.","journal-title":"Real Estate Econ"},{"key":"1028_CR87","doi-asserted-by":"crossref","unstructured":"Lu Q, Zhu L, Xu X, Whittle J, Zowghi D, Jacquet A. Responsible ai pattern catalogue: A collection of best practices for ai governance and engineering. ACM Comput Surveys. 2023.","DOI":"10.1145\/3626234"},{"key":"1028_CR88","doi-asserted-by":"publisher","DOI":"10.4855\/arXiv.2301.05517","author":"Q Lu","year":"2023","unstructured":"Lu Q, Luo Y, Zhu L, Tang M, Xu X, Whittle J. Developing responsible chatbots for financial services: a pattern-oriented responsible AI engineering approach. IEEE Intell Syst. 2023. https:\/\/doi.org\/10.4855\/arXiv.2301.05517.","journal-title":"IEEE Intell Syst"},{"key":"1028_CR89","first-page":"100","volume-title":"Deepmutation: mutation testing of deep learning systems","author":"L Ma","year":"2018","unstructured":"Ma L, Zhang F, Sun J, Xue M, Li B, Juefei-Xu F, Xie C, Li L, Liu Y, Zhao J, Wang Y. Deepmutation: mutation testing of deep learning systems. Piscataway: IEEE; 2018. p. 100\u201311."},{"key":"1028_CR90","doi-asserted-by":"publisher","DOI":"10.4855\/arXiv.2012.01805","author":"R Marcinkevi\u010ds","year":"2020","unstructured":"Marcinkevi\u010ds R, Vogt JE. Interpretability and explainability: a machine learning zoo mini-tour. arXiv preprint. 2020. https:\/\/doi.org\/10.4855\/arXiv.2012.01805.","journal-title":"arXiv preprint"},{"key":"1028_CR91","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1007\/978-1-4613-9080-0_1","volume-title":"Computers, chess, and cognition","author":"TA Marsland","year":"1990","unstructured":"Marsland TA. A short history of computer chess. In: Anthony Marsland T, Schaeffer J, editors. Computers, chess, and cognition. New York: Springer; 1990. p. 3\u20137."},{"key":"1028_CR92","volume-title":"TensorFlow machine learning cookbook","author":"N McClure","year":"2017","unstructured":"McClure N. TensorFlow machine learning cookbook. Birmingham: PACKT publishing Ltd.; 2017."},{"key":"1028_CR93","volume-title":"Data driven approach towards disruptive technologies: proceedings of MIDAS 2020","author":"T Mehrotra","year":"2021","unstructured":"Mehrotra T, Rajput GK, Verma M, Lakhani B, Singh N. Email spam filtering technique from various perspectives using machine learning algorithms. In: Singh TP, Tomar R, Choudhury T, Perumal T, Mahdi HF, editors. Data driven approach towards disruptive technologies: proceedings of MIDAS 2020. Singapore: Springer; 2021."},{"key":"1028_CR94","volume-title":"The turing test: the elusive standard of artificial intelligence","year":"2003","unstructured":"Moor J, editor. The turing test: the elusive standard of artificial intelligence. Dordrecht: Springer Science & Business Media; 2003."},{"key":"1028_CR95","doi-asserted-by":"publisher","DOI":"10.4855\/arXiv.2401.11131","author":"RK Mothilal","year":"2024","unstructured":"Mothilal RK, Guha S, Ahmed SI. Towards a non-ideal methodological framework for responsible ML. arXiv preprint. 2024. https:\/\/doi.org\/10.4855\/arXiv.2401.11131.","journal-title":"arXiv preprint"},{"issue":"3","key":"1028_CR96","doi-asserted-by":"publisher","first-page":"e0299932","DOI":"10.1371\/journal.pone.0299932","volume":"19","author":"T Mroz","year":"2024","unstructured":"Mroz T, Griffin M, Cartabuke R, Laffin L, Russo-Alvarez G, Thomas G, Smedira N, Meese T, Shost M, Habboub G. Predicting hypertension control using machine learning. PLoS ONE. 2024;19(3):e0299932.","journal-title":"PLoS ONE"},{"key":"1028_CR97","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s40411-017-0037-x","volume":"5","author":"J Murillo-Morera","year":"2017","unstructured":"Murillo-Morera J, Quesada-L\u00f3pez C, Castro-Herrera C, Jenkins M. A genetic algorithm based framework for software effort prediction. J Softw Eng Res Dev. 2017;5:1\u201333.","journal-title":"J Softw Eng Res Dev"},{"key":"1028_CR98","unstructured":"Murphy C, Kaiser GE, Hu L. Properties of machine learning applications for use in metamorphic testing."},{"key":"1028_CR99","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s40537-014-0007-7","volume":"2","author":"MM Najafabadi","year":"2015","unstructured":"Najafabadi MM, Villanustre F, Khoshgoftaar TM, Seliya N, Wald R, Muharemagic E. Deep learning applications and challenges in big data analytics. J Big Data. 2015;2:1\u201321.","journal-title":"J Big Data"},{"key":"1028_CR100","doi-asserted-by":"crossref","unstructured":"Nie Y, Wang Y, Bansal M. Analyzing compositionality-sensitivity of NLI models. In Proceedings of the AAAI conference on artificial intelligence 2019 Jul 17, Vol. 33, No. 01. pp. 6867\u20136874.","DOI":"10.1609\/aaai.v33i01.33016867"},{"key":"1028_CR101","first-page":"273","volume-title":"A test architecture for machine learning product","author":"Y Nishi","year":"2018","unstructured":"Nishi Y, Masuda S, Ogawa H, Uetsuki K. A test architecture for machine learning product. Piscataway: IEEE; 2018. p. 273\u20138."},{"key":"1028_CR102","first-page":"472","volume-title":"Explicit control of the level of expressiveness in DNN-based speech synthesis by embedding interpolation","author":"T Nosek","year":"2021","unstructured":"Nosek T, Suzi\u0107 S, Vujovi\u0107 M, Pekar D, Se\u010dujski M, Deli\u0107 V. Explicit control of the level of expressiveness in DNN-based speech synthesis by embedding interpolation. St. Petersburg: Springer International Publishing; 2021. p. 472\u201382."},{"issue":"2","key":"1028_CR103","doi-asserted-by":"publisher","first-page":"81","DOI":"10.26599\/BDMA.2021.9020028","volume":"5","author":"IK Nti","year":"2022","unstructured":"Nti IK, Quarcoo JA, Aning J, Fosu GK. A mini-review of machine learning in big data analytics: applications, challenges, and prospects. Big Data Min Anal. 2022;5(2):81\u201397.","journal-title":"Big Data Min Anal"},{"key":"1028_CR104","doi-asserted-by":"publisher","DOI":"10.2504\/aj0505114","author":"D Nurseitov","year":"2021","unstructured":"Nurseitov D, Bostanbekov K, Kanatov M, Alimova A, Abdallah A, Abdimanap G. Classification of handwritten names of cities and handwritten text recognition using various deep learning models. ArXiv preprint. 2021. https:\/\/doi.org\/10.2504\/aj0505114.","journal-title":"ArXiv preprint"},{"key":"1028_CR105","unstructured":"Odena A, Olsson C, Andersen D, Goodfellow I. Tensorfuzz: Debugging neural networks with coverage-guided fuzzing. InInternational Conference on Machine Learning 2019 May 24. PMLR. pp. 4901\u20134911."},{"issue":"23","key":"1028_CR106","doi-asserted-by":"publisher","first-page":"5191","DOI":"10.3390\/app9235191","volume":"9","author":"S Oh","year":"2019","unstructured":"Oh S. Feature interaction in terms of prediction performance. Appl Sci. 2019;9(23):5191.","journal-title":"Appl Sci"},{"key":"1028_CR107","doi-asserted-by":"publisher","first-page":"155","DOI":"10.1007\/978-3-030-43883-8_7","volume-title":"Recent trends in learning from data: tutorials from the inns big data and deep learning conference (innsbddl2019)","author":"L Oneto","year":"2020","unstructured":"Oneto L, Chiappa S. Fairness in machine learning. In: Oneto L, Navarin N, Sperduti A, Anguita D, editors. Recent trends in learning from data: tutorials from the inns big data and deep learning conference (innsbddl2019). Cham: Springer International Publishing; 2020. p. 155\u201396."},{"key":"1028_CR108","doi-asserted-by":"crossref","unstructured":"Papineni K, Roukos S, Ward T, Zhu WJ. Bleu: a method for automatic evaluation of machine translation. In: Proceedings of the 40th annual meeting of the Association for Computational Linguistics 2002 Jul. pp. 311\u2013318.","DOI":"10.3115\/1073083.1073135"},{"key":"1028_CR109","doi-asserted-by":"crossref","unstructured":"Pei K, Cao Y, Yang J, Jana S. Deepxplore: Automated whitebox testing of deep learning systems. In: proceedings of the 26th Symposium on Operating Systems Principles 2017 Oct 14. pp. 1\u201318.","DOI":"10.1145\/3132747.3132785"},{"key":"1028_CR110","first-page":"1027","volume-title":"CRADLE: cross-backend validation to detect and localize bugs in deep learning libraries","author":"HV Pham","year":"2019","unstructured":"Pham HV, Lutellier T, Qi W, Tan L. CRADLE: cross-backend validation to detect and localize bugs in deep learning libraries. Piscataway: IEEE; 2019. p. 1027\u201338."},{"issue":"1","key":"1028_CR111","first-page":"334","volume":"15","author":"N Polyzotis","year":"2019","unstructured":"Polyzotis N, Zinkevich M, Roy S, Breck E, Whang S. Data validation for machine learning. Proc Mach Learn Syst. 2019;15(1):334\u201347.","journal-title":"Proc Mach Learn Syst"},{"key":"1028_CR112","first-page":"22","volume-title":"Quality assurance for machine learning\u2013an approach to function and system safeguarding","author":"A Poth","year":"2020","unstructured":"Poth A, Meyer B, Schlicht P, Riel A. Quality assurance for machine learning\u2013an approach to function and system safeguarding. Piscataway: IEEE; 2020. p. 22\u20139."},{"issue":"16","key":"1028_CR113","doi-asserted-by":"publisher","first-page":"24365","DOI":"10.1007\/s11042-021-10707-4","volume":"80","author":"M Puttagunta","year":"2021","unstructured":"Puttagunta M, Ravi S. Medical image analysis based on deep learning approach. Multimed Tools Appl. 2021;80(16):24365\u201398.","journal-title":"Multimed Tools Appl"},{"key":"1028_CR114","doi-asserted-by":"publisher","DOI":"10.4855\/arXiv.1803.06071","author":"Z Qi","year":"2018","unstructured":"Qi Z, Wang H, Li J, Gao H. Impacts of dirty data: and experimental evaluation. arXiv preprint. 2018. https:\/\/doi.org\/10.4855\/arXiv.1803.06071.","journal-title":"arXiv preprint"},{"key":"1028_CR115","doi-asserted-by":"publisher","first-page":"25","DOI":"10.1007\/s10489-006-0001-7","volume":"26","author":"B Qian","year":"2007","unstructured":"Qian B, Rasheed K. Stock market prediction with multiple classifiers. Appl Intell. 2007;26:25\u201333.","journal-title":"Appl Intell"},{"key":"1028_CR116","first-page":"171","volume-title":"Syneva: evaluating ml programs by mirror program synthesis","author":"Y Qin","year":"2018","unstructured":"Qin Y, Wang H, Xu C, Ma X, Lu J. Syneva: evaluating ml programs by mirror program synthesis. Piscataway: IEEE; 2018. p. 171\u201382."},{"issue":"10","key":"1028_CR117","doi-asserted-by":"publisher","first-page":"11029","DOI":"10.1007\/s11227-023-05094-6","volume":"79","author":"M Qorich","year":"2023","unstructured":"Qorich M, El Ouazzani R. Text sentiment classification of Amazon reviews using word embeddings and convolutional neural networks. J Supercomput. 2023;79(10):11029\u201354.","journal-title":"J Supercomput"},{"key":"1028_CR118","first-page":"786","volume-title":"Integrating symbolic and statistical methods for testing intelligent systems: applications to machine learning and computer vision","author":"A Ramanathan","year":"2016","unstructured":"Ramanathan A, Pullum LL, Hussain F, Chakrabarty D, Jha SK. Integrating symbolic and statistical methods for testing intelligent systems: applications to machine learning and computer vision. Piscataway: IEEE; 2016. p. 786\u201391."},{"issue":"23","key":"1028_CR119","doi-asserted-by":"publisher","first-page":"100178","DOI":"10.1016\/j.bdr.2020.100178","volume":"15","author":"X Ren","year":"2021","unstructured":"Ren X, Li X, Ren K, Song J, Xu Z, Deng K, Wang X. Deep learning-based weather prediction: a survey. Big Data Res. 2021;15(23):100178.","journal-title":"Big Data Res"},{"key":"1028_CR120","doi-asserted-by":"publisher","first-page":"5193","DOI":"10.1007\/s10664-020-09881-0","volume":"25","author":"V Riccio","year":"2020","unstructured":"Riccio V, Jahangirova G, Stocco A, Humbatova N, Weiss M, Tonella P. Testing machine learning based systems: a systematic mapping. Empir Softw Eng. 2020;25:5193\u2013254.","journal-title":"Empir Softw Eng"},{"key":"1028_CR121","doi-asserted-by":"crossref","unstructured":"Rismani S, Shelby R, Smart A, Jatho E, Kroll J, Moon A, Rostamzadeh N. From plane crashes to algorithmic harm: applicability of safety engineering frameworks for responsible ML. In: Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems 2023 Apr 19. pp. 1\u201318.","DOI":"10.1145\/3544548.3581407"},{"key":"1028_CR122","volume-title":"Build a career in data science","author":"E Robinson","year":"2020","unstructured":"Robinson E, Nolis J. Build a career in data science. Shelter Island: Manning Publications; 2020."},{"issue":"1","key":"1028_CR123","doi-asserted-by":"publisher","first-page":"2166719","DOI":"10.1080\/08839514.2023.2166719","volume":"37","author":"H Saleh","year":"2023","unstructured":"Saleh H, Alhothali A, Moria K. Detection of hate speech using bert and hate speech word embedding with deep model. Appl Artif Intell. 2023;37(1):2166719.","journal-title":"Appl Artif Intell"},{"key":"1028_CR124","doi-asserted-by":"publisher","DOI":"10.1109\/CAI59869.2024.00215","author":"C Sanderson","year":"2024","unstructured":"Sanderson C, Schleiger E, Douglas D, Kuhnert P, Lu Q. Resolving ethics trade-offs in implementing responsible AI. arXiv preprint. 2024. https:\/\/doi.org\/10.1109\/CAI59869.2024.00215.","journal-title":"arXiv preprint"},{"key":"1028_CR125","volume-title":"On the move to meaningful internet systems. OTM 2018 conferences: confederated international conferences: CoopIS, C&TC, and ODBASE 2018, Valletta, Malta, October 22-26, 2018, Proceedings, Part I","author":"S Satyal","year":"2018","unstructured":"Satyal S, Weber I, Paik HY, Di Ciccio C, Mendling J. Shadow testing for business process improvement. In: Panetto H, Debruyne C, Proper HA, Ardagna CA, Roman D, Meersman R, editors. On the move to meaningful internet systems. OTM 2018 conferences: confederated international conferences: CoopIS, C&TC, and ODBASE 2018, Valletta, Malta, October 22-26, 2018, Proceedings, Part I. Cham: Springer International Publishing; 2018."},{"issue":"1\u20132","key":"1028_CR126","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/S0004-3702(01)00165-5","volume":"134","author":"J Schaeffer","year":"2002","unstructured":"Schaeffer J, Van den Herik HJ. Games, computers, and artificial intelligence. Artif Intell. 2002;134(1\u20132):1\u20137.","journal-title":"Artif Intell"},{"key":"1028_CR127","unstructured":"Selsam D, Liang P, Dill DL. Developing bug-free machine learning systems with formal mathematics. In: International Conference on Machine Learning 2017 Jul 17. PMLR. pp. 3047\u20133056."},{"key":"1028_CR128","first-page":"528","volume-title":"Predicting solar generation from weather forecasts using machine learning","author":"N Sharma","year":"2011","unstructured":"Sharma N, Sharma P, Irwin D, Shenoy P. Predicting solar generation from weather forecasts using machine learning. Piscataway: IEEE; 2011. p. 528\u201333."},{"key":"1028_CR129","doi-asserted-by":"publisher","DOI":"10.4855\/arXiv.1907.09427","author":"S Sherin","year":"2019","unstructured":"Sherin S, Iqbal MZ. A systematic mapping study on testing of machine learning programs. arXiv preprint. 2019. https:\/\/doi.org\/10.4855\/arXiv.1907.09427.","journal-title":"arXiv preprint"},{"issue":"15","key":"1028_CR130","doi-asserted-by":"publisher","first-page":"105","DOI":"10.1007\/s12519-019-00255-1","volume":"1","author":"LQ Shu","year":"2019","unstructured":"Shu LQ, Sun YK, Tan LH, Shu Q, Chang AC. Application of artificial intelligence in pediatrics: past, present and future. World J Pediatrics. 2019;1(15):105\u20138.","journal-title":"World J Pediatrics"},{"key":"1028_CR131","first-page":"17","volume-title":"Towards guidelines for assessing qualities of machine learning systems","author":"J Siebert","year":"2020","unstructured":"Siebert J, Joeckel L, Heidrich J, Nakamichi K, Ohashi K, Namba I, Yamamoto R, Aoyama M. Towards guidelines for assessing qualities of machine learning systems. Cham: Springer International Publishing; 2020. p. 17\u201331."},{"issue":"2","key":"1028_CR132","doi-asserted-by":"publisher","first-page":"307","DOI":"10.1007\/s11219-021-09557-y","volume":"30","author":"J Siebert","year":"2022","unstructured":"Siebert J, Joeckel L, Heidrich J, Trendowicz A, Nakamichi K, Ohashi K, Namba I, Yamamoto R, Aoyama M. Construction of a quality model for machine learning systems. Softw Qual J. 2022;30(2):307\u201335.","journal-title":"Softw Qual J"},{"key":"1028_CR133","doi-asserted-by":"publisher","DOI":"10.4855\/arXiv.2201.05647","author":"R Soklaski","year":"2022","unstructured":"Soklaski R, Goodwin J, Brown O, Yee M, Matterer J. Tools and practices for responsible AI engineering. arXiv preprint. 2022. https:\/\/doi.org\/10.4855\/arXiv.2201.05647.","journal-title":"arXiv preprint"},{"key":"1028_CR134","doi-asserted-by":"crossref","unstructured":"Song Q, Borg M, Engstr\u00f6m E, Ard\u00f6 H, Rico S. Exploring ML testing in practice: Lessons learned from an interactive rapid review with axis communications. In: Proceedings of the 1st International Conference on AI Engineering: Software Engineering for AI 2022 May 16. pp. 10\u201321.","DOI":"10.1145\/3522664.3528596"},{"key":"1028_CR135","doi-asserted-by":"crossref","unstructured":"Sparks ER, Talwalkar A, Haas D, Franklin MJ, Jordan MI, Kraska T. Automating model search for large scale machine learning. In: Proceedings of the Sixth ACM Symposium on Cloud Computing 2015 Aug 27. pp. 368\u2013380.","DOI":"10.1145\/2806777.2806945"},{"key":"1028_CR136","first-page":"1","volume-title":"Evaluation of automatic face recognition for automatic border control on actual data recorded of travellers at Schiphol Airport","author":"LJ Spreeuwers","year":"2012","unstructured":"Spreeuwers LJ, Hendrikse AJ, Gerritsen KJ. Evaluation of automatic face recognition for automatic border control on actual data recorded of travellers at Schiphol Airport. Piscataway: IEEE; 2012. p. 1\u20136."},{"key":"1028_CR137","unstructured":"Srisakaokul S, Wu Z, Astorga A, Alebiosu O, Xie T. Multiple-implementation testing of supervised learning software. In: Workshops at the thirty-second AAAI conference on artificial intelligence 2018 Jun 20."},{"issue":"4","key":"1028_CR138","first-page":"87","volume":"3","author":"PR Srivastava","year":"2009","unstructured":"Srivastava PR, Kim TH. Application of genetic algorithm in software testing. Int J Softw Eng Appl. 2009;3(4):87\u201396.","journal-title":"Int J Softw Eng Appl"},{"issue":"3","key":"1028_CR139","first-page":"032004","volume":"32","author":"M Stanojevi\u0107","year":"2023","unstructured":"Stanojevi\u0107 M, Dra\u0161kovi\u0107 D, Nikoli\u0107 B. Retinal disease classification based on optical coherence tomography images using convolutional neural networks. J Electron Imaging. 2023;32(3):032004.","journal-title":"J Electron Imaging"},{"issue":"199","key":"1028_CR140","doi-asserted-by":"publisher","first-page":"111615","DOI":"10.1016\/j.jss.2023.111615","volume":"1","author":"M Steidl","year":"2023","unstructured":"Steidl M, Felderer M, Ramler R. The pipeline for the continuous development of artificial intelligence models\u2014current state of research and practice. J Syst Softw. 2023;1(199):111615.","journal-title":"J Syst Softw"},{"key":"1028_CR141","first-page":"1647","volume-title":"Big-data-driven anomaly detection in industry (4.0): an approach and a case study","author":"L Stojanovic","year":"2016","unstructured":"Stojanovic L, Dinic M, Stojanovic N, Stojadinovic A. Big-data-driven anomaly detection in industry (4.0): an approach and a case study. Piscataway: IEEE; 2016. p. 1647\u201352."},{"key":"1028_CR142","unstructured":"Strasser N. An evaluation of canary deployment tools. Doctoral dissertation, University of Applied Sciences; 2023."},{"issue":"2","key":"1028_CR143","doi-asserted-by":"publisher","first-page":"392","DOI":"10.3390\/make3020020","volume":"3","author":"S Studer","year":"2021","unstructured":"Studer S, Bui TB, Drescher C, Hanuschkin A, Winkler L, Peters S, M\u00fcller KR. Towards CRISP-ML (Q): a machine learning process model with quality assurance methodology. Mach Learn Knowl Extr. 2021;3(2):392\u2013413.","journal-title":"Mach Learn Knowl Extr"},{"key":"1028_CR144","unstructured":"Sugali K. Software testing: Issues and challenges of artificial intelligence & machine learning."},{"key":"1028_CR145","doi-asserted-by":"crossref","unstructured":"Sun Z, Zhang JM, Harman M, Papadakis M, Zhang L. Automatic testing and improvement of machine translation. In: Proceedings of the ACM\/IEEE 42nd international conference on software engineering 2020 Jun 27. pp. 974\u2013985.","DOI":"10.1145\/3377811.3380420"},{"issue":"1","key":"1028_CR146","doi-asserted-by":"publisher","first-page":"1","DOI":"10.4018\/JCIT.2021010101","volume":"23","author":"F Tahiru","year":"2021","unstructured":"Tahiru F. AI in education: A systematic literature review. J Cases Inform Technol (JCIT). 2021;23(1):1\u201320.","journal-title":"J Cases Inform Technol (JCIT)"},{"issue":"1","key":"1028_CR147","doi-asserted-by":"publisher","first-page":"76","DOI":"10.1186\/s40537-021-00468-0","volume":"8","author":"I Taleb","year":"2021","unstructured":"Taleb I, Serhani MA, Bouhaddioui C, Dssouli R. Big data quality framework: a holistic approach to continuous quality management. J Big Data. 2021;8(1):76.","journal-title":"J Big Data"},{"key":"1028_CR148","first-page":"649","volume-title":"Big data transformation testing based on data reverse engineering","author":"DG Tesfagiorgish","year":"2015","unstructured":"Tesfagiorgish DG, JunYi L. Big data transformation testing based on data reverse engineering. Piscataway: IEEE; 2015. p. 649\u201352."},{"key":"1028_CR149","first-page":"271","volume-title":"An empirical study of bugs in machine learning systems","author":"F Thung","year":"2012","unstructured":"Thung F, Wang S, Lo D, Jiang L. An empirical study of bugs in machine learning systems. Piscataway: IEEE; 2012. p. 271\u201380."},{"key":"1028_CR150","doi-asserted-by":"crossref","unstructured":"Tian Y, Pei K, Jana S, Ray B. Deeptest: Automated testing of deep-neural-network-driven autonomous cars. In: Proceedings of the 40th international conference on software engineering 2018 May 27. pp. 303\u2013314.","DOI":"10.1145\/3180155.3180220"},{"key":"1028_CR151","doi-asserted-by":"crossref","unstructured":"[Toslali2021] Toslali M, Parthasarathy S, Oliveira F, Huang H, Coskun AK. Iter8: Online experimentation in the cloud. In: Proceedings of the ACM Symposium on Cloud Computing 2021 Nov 1. pp. 289\u2013304.","DOI":"10.1145\/3472883.3486984"},{"issue":"11","key":"1028_CR152","doi-asserted-by":"publisher","first-page":"3024","DOI":"10.1109\/ACCESS.2023.3234438","volume":"5","author":"IA Vajs","year":"2023","unstructured":"Vajs IA, Kva\u0161\u010dev GS, Papi\u0107 TM, Jankovi\u0107 MM. Eye-tracking image encoding: Autoencoders for the crossing of language boundaries in developmental dyslexia detection. IEEE Access. 2023;5(11):3024\u201333.","journal-title":"IEEE Access"},{"key":"1028_CR153","first-page":"1","volume-title":"Engineering safety in machine learning","author":"KR Varshney","year":"2016","unstructured":"Varshney KR. Engineering safety in machine learning. Piscataway: IEEE; 2016. p. 1\u20135."},{"key":"1028_CR154","doi-asserted-by":"crossref","unstructured":"Verma S, Rubin J. Fairness definitions explained. In: Proceedings of the international workshop on software fairness 2018 May 29. pp. 1\u20137.","DOI":"10.1145\/3194770.3194776"},{"issue":"62","key":"1028_CR155","doi-asserted-by":"publisher","first-page":"101","DOI":"10.1016\/j.infsof.2015.02.009","volume":"1","author":"S Wagner","year":"2015","unstructured":"Wagner S, Goeb A, Heinemann L, Kl\u00e4s M, Lampasona C, Lochmann K, Mayr A, Pl\u00f6sch R, Seidl A, Streit J, Trendowicz A. Operationalised product quality models and assessment: the Quamoco approach. Inf Softw Technol. 2015;1(62):101\u201323.","journal-title":"Inf Softw Technol"},{"key":"1028_CR156","doi-asserted-by":"publisher","first-page":"1400","DOI":"10.1007\/978-3-540-24855-2_160","volume-title":"Genetic and evolutionary computation\u2014GECCO 2004: genetic and evolutionary computation conference, Seattle, WA, USA, June 26-30, 2004. Proceedings, part II","author":"J Wegener","year":"2004","unstructured":"Wegener J, B\u00fchler O. Evaluation of different fitness functions for the evolutionary testing of an autonomous parking system. In: Deb K, editor. Genetic and evolutionary computation\u2014GECCO 2004: genetic and evolutionary computation conference, Seattle, WA, USA, June 26-30, 2004. Proceedings, part II. Berlin: Springer; 2004. p. 1400\u201312."},{"key":"1028_CR157","unstructured":"Werpachowski R, Gy\u00f6rgy A, Szepesv\u00e1ri C. Detecting overfitting via adversarial examples. Adv Neural Infor Process Syst. 2019;32."},{"issue":"12","key":"1028_CR158","doi-asserted-by":"publisher","first-page":"8209","DOI":"10.1007\/s00500-022-07716-2","volume":"27","author":"JL Wu","year":"2023","unstructured":"Wu JL, Tang XR, Hsu CH. A prediction model of stock market trading actions using generative adversarial network and piecewise linear representation approaches. Soft Comput. 2023;27(12):8209\u201322.","journal-title":"Soft Comput"},{"key":"1028_CR159","doi-asserted-by":"publisher","DOI":"10.4855\/arXiv.2108.04417","author":"R Xu","year":"2021","unstructured":"Xu R, Baracaldo N, Joshi J. Privacy-preserving machine learning: methods, challenges and directions. arXiv preprint. 2021. https:\/\/doi.org\/10.4855\/arXiv.2108.04417.","journal-title":"arXiv preprint"},{"key":"1028_CR160","doi-asserted-by":"crossref","unstructured":"Yaghoubi S, Fainekos G. Gray-box adversarial testing for control systems with machine learning components. In: Proceedings of the 22nd ACM International Conference on Hybrid Systems: Computation and Control 2019 Apr 16. pp. 179\u2013184.","DOI":"10.1145\/3302504.3311814"},{"key":"1028_CR161","first-page":"185","volume-title":"Service discovery based blue-green deployment technique in cloud native environments","author":"B Yang","year":"2018","unstructured":"Yang B, Sailer A, Jain S, Tomala-Reyes AE, Singh M, Ramnath A. Service discovery based blue-green deployment technique in cloud native environments. Piscataway: IEEE; 2018. p. 185\u201392."},{"key":"1028_CR162","first-page":"69","volume-title":"Service-oriented computing\u2014ICSOC 2019 workshops: WESOACS, ASOCA, ISYCC, TBCE, and STRAPS, Toulouse, France, October 28\u201331, 2019, Revised Selected Papers","author":"B Yang","year":"2019","unstructured":"Yang B, Sailer A, Mohindra A. Survey and evaluation of blue-green deployment techniques in cloud native environments. In: Yangui S, Bouguettaya A, Xue X, Faci N, Gaaloul W, Qi Y, Zhou Z, Hernandez N, Nakagawa EY, editors. Service-oriented computing\u2014ICSOC 2019 workshops: WESOACS, ASOCA, ISYCC, TBCE, and STRAPS, Toulouse, France, October 28\u201331, 2019, Revised Selected Papers. Cham: Springer International Publishing; 2019. p. 69\u201381."},{"issue":"17","key":"1028_CR163","first-page":"1","volume":"17","author":"K Yessenov","year":"2009","unstructured":"Yessenov K, Misailovic S. Sentiment analysis of movie review comments. Methodology. 2009;17(17):1\u20137.","journal-title":"Methodology"},{"key":"1028_CR164","doi-asserted-by":"crossref","unstructured":"Yesudas M, S GM, Nair SK. High-volume performance test framework using big data. In: Proceedings of the 4th International Workshop on Large-Scale Testing 2015 Feb 1. pp. 13\u201316.","DOI":"10.1145\/2693182.2693185"},{"key":"1028_CR165","doi-asserted-by":"crossref","unstructured":"Zhang M, Zhang Y, Zhang L, Liu C, Khurshid S. Deeproad: Gan-based metamorphic testing and input validation framework for autonomous driving systems. In: Proceedings of the 33rd ACM\/IEEE International Conference on Automated Software Engineering 2018 Sep 3. pp. 132\u2013142.","DOI":"10.1145\/3238147.3238187"},{"issue":"1","key":"1028_CR166","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/TSE.2019.2962027","volume":"48","author":"JM Zhang","year":"2020","unstructured":"Zhang JM, Harman M, Ma L, Liu Y. Machine learning testing: survey, landscapes and horizons. IEEE Trans Software Eng. 2020;48(1):1\u201336.","journal-title":"IEEE Trans Software Eng"},{"key":"1028_CR167","first-page":"494","volume-title":"Towards mlops: a case study of ml pipeline platform","author":"Y Zhou","year":"2020","unstructured":"Zhou Y, Yu Y, Ding B. Towards mlops: a case study of ml pipeline platform. Piscataway: IEEE; 2020. p. 494\u2013500."}],"container-title":["Journal of Big Data"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1186\/s40537-024-01028-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1186\/s40537-024-01028-y\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1186\/s40537-024-01028-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,30]],"date-time":"2024-10-30T17:05:07Z","timestamp":1730307907000},"score":1,"resource":{"primary":{"URL":"https:\/\/journalofbigdata.springeropen.com\/articles\/10.1186\/s40537-024-01028-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,30]]},"references-count":167,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2024,12]]}},"alternative-id":["1028"],"URL":"https:\/\/doi.org\/10.1186\/s40537-024-01028-y","relation":{},"ISSN":["2196-1115"],"issn-type":[{"value":"2196-1115","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,10,30]]},"assertion":[{"value":"2 May 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 October 2024","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 October 2024","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"Not applicable.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics approval and consent to participate"}},{"value":"The authors declare no competing interests.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"156"}}