{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,10]],"date-time":"2026-04-10T07:46:59Z","timestamp":1775807219110,"version":"3.50.1"},"publisher-location":"Cham","reference-count":37,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030658533","type":"print"},{"value":"9783030658540","type":"electronic"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-65854-0_6","type":"book-chapter","created":{"date-parts":[[2021,1,6]],"date-time":"2021-01-06T00:13:06Z","timestamp":1609891986000},"page":"66-77","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":12,"title":["The AIQ Meta-Testbed: Pragmatically Bridging Academic AI Testing and Industrial Q Needs"],"prefix":"10.1007","author":[{"given":"Markus","family":"Borg","sequence":"first","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2021,1,6]]},"reference":[{"key":"6_CR1","volume-title":"Driverless: Intelligent Cars and the Road Ahead","author":"H Lipson","year":"2016","unstructured":"Lipson, H., Kurman, M.: Driverless: Intelligent Cars and the Road Ahead. MIT Press, Cambridge (2016)"},{"issue":"4","key":"6_CR2","doi-asserted-by":"publisher","first-page":"230","DOI":"10.1136\/svn-2017-000101","volume":"2","author":"F Jiang","year":"2017","unstructured":"Jiang, F., et al.: Artificial intelligence in healthcare: past, present and future. Stroke Vasc. Neurol. 2(4), 230\u2013243 (2017)","journal-title":"Stroke Vasc. Neurol."},{"key":"6_CR3","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-64822-4","volume-title":"Software Quality Assurance: Consistency in the Face of Complexity and Change","author":"N Walkinshaw","year":"2017","unstructured":"Walkinshaw, N.: Software Quality Assurance: Consistency in the Face of Complexity and Change. Springer, Heidelberg (2017). https:\/\/doi.org\/10.1007\/978-3-319-64822-4"},{"issue":"1","key":"6_CR4","doi-asserted-by":"publisher","first-page":"1","DOI":"10.2991\/jase.d.190131.001","volume":"1","author":"M Borg","year":"2019","unstructured":"Borg, M., et al.: Safely entering the deep: a review of verification and validation for machine learning and a challenge elicitation in the automotive industry. J. Autom. Softw. Eng. 1(1), 1\u201319 (2019)","journal-title":"J. Autom. Softw. Eng."},{"key":"6_CR5","doi-asserted-by":"crossref","unstructured":"Salay, R., Queiroz, R., Czarnecki, K.: An Analysis of ISO 26262: Machine Learning and Safety in Automotive Software. SAE Technical Paper 2018\u201301-1075 (2018)","DOI":"10.4271\/2018-01-1075"},{"key":"6_CR6","first-page":"25","volume":"20","author":"A Azulay","year":"2019","unstructured":"Azulay, A., Weiss, Y.: Why do deep convolutional networks generalize so poorly to small image transformations? J. Mach. Learn. Res. 20, 25 (2019)","journal-title":"J. Mach. Learn. Res."},{"key":"6_CR7","volume-title":"Handbook Of Software Quality Assurance","author":"G Schulmeyer","year":"1987","unstructured":"Schulmeyer, G.: Handbook Of Software Quality Assurance, 1st edn. Prentice Hall, Upper Saddle River (1987)","edition":"1"},{"key":"6_CR8","volume-title":"Software Quality Assurance: From Theory to Implementation","author":"D Galin","year":"2003","unstructured":"Galin, D.: Software Quality Assurance: From Theory to Implementation. Pearson, Harlow (2003)"},{"key":"6_CR9","volume-title":"Software Quality Assurance: In Large Scale and Complex Software-Intensive Systems","year":"2016","unstructured":"Mistrik, I., Soley, R.M., Ali, N., Grundy, J., Tekinerdogan, B. (eds.): Software Quality Assurance: In Large Scale and Complex Software-Intensive Systems. Morgan Kaufmann, Waltham (2016)"},{"key":"6_CR10","doi-asserted-by":"publisher","first-page":"52138","DOI":"10.1109\/ACCESS.2018.2870052","volume":"6","author":"A Adadi","year":"2018","unstructured":"Adadi, A., Berrada, M.: Peeking inside the black-box: a survey on explainable artificial intelligence (XAI). IEEE Access 6, 52138\u201352160 (2018)","journal-title":"IEEE Access"},{"key":"6_CR11","unstructured":"Borg, M.: Explainability first! Cousteauing the depths of neural networks to argue safety. In: Greenyer, J., Lochau, M., Vogel, T., (eds.) Explainable Software for Cyber-Physical Systems (ES4CPS): Report from the GI Dagstuhl Seminar 19023, pp. 26\u201327 (2019)"},{"key":"6_CR12","doi-asserted-by":"crossref","unstructured":"Vogelsang, A., Borg, M.: Requirements engineering for machine learning: perspectives from data scientists. In: Proceedings of the 27th International Requirements Engineering Conference Workshops, pp. 245\u2013251 (2019)","DOI":"10.1109\/REW.2019.00050"},{"key":"6_CR13","doi-asserted-by":"crossref","unstructured":"Weyns, D., et al.: A survey of formal methods in self-adaptive systems. In: Proceedings of the 5th International Conference on Computer Science and Software Engineering, pp. 67\u201379 (2012)","DOI":"10.1145\/2347583.2347592"},{"issue":"8","key":"6_CR14","doi-asserted-by":"publisher","first-page":"821","DOI":"10.1016\/j.infsof.2014.03.003","volume":"56","author":"CA Gonzalez","year":"2014","unstructured":"Gonzalez, C.A., Cabot, J.: Formal verification of static software models in MDE: a systematic review. Inf. Softw. Tech. 56(8), 821\u2013838 (2014)","journal-title":"Inf. Softw. Tech."},{"issue":"6","key":"6_CR15","doi-asserted-by":"publisher","first-page":"30","DOI":"10.1145\/255656.255692","volume":"40","author":"J Herbsleb","year":"1997","unstructured":"Herbsleb, J., et al.: Software quality and the capability maturity model. Commun. ACM 40(6), 30\u201340 (1997)","journal-title":"Commun. ACM"},{"issue":"7","key":"6_CR16","doi-asserted-by":"publisher","first-page":"677","DOI":"10.1016\/S0378-7206(02)00096-4","volume":"40","author":"N Ashrafi","year":"2003","unstructured":"Ashrafi, N.: The impact of software process improvement on quality: theory and practice. Inf. Manag. 40(7), 677\u2013690 (2003)","journal-title":"Inf. Manag."},{"issue":"6","key":"6_CR17","doi-asserted-by":"publisher","first-page":"687","DOI":"10.1145\/62959.62965","volume":"31","author":"D Gelperin","year":"1988","unstructured":"Gelperin, D., Hetzel, B.: The growth of software testing. Commun. ACM 31(6), 687\u2013695 (1988)","journal-title":"Commun. ACM"},{"key":"6_CR18","doi-asserted-by":"crossref","unstructured":"Orso, A., Rothermel, G.: Software testing: a research travelogue (2000\u20132014). In: Future of Software Engineering Proceedings, pp. 117\u2013132 (2014)","DOI":"10.1145\/2593882.2593885"},{"issue":"5","key":"6_CR19","doi-asserted-by":"publisher","first-page":"46","DOI":"10.1109\/MS.2017.3571582","volume":"34","author":"M Kassab","year":"2017","unstructured":"Kassab, M., DeFranco, J.F., Laplante, P.A.: Software testing: the state of the practice. IEEE Softw. 34(5), 46\u201352 (2017)","journal-title":"IEEE Softw."},{"key":"6_CR20","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4842-3432-7","volume-title":"Building Intelligent Systems: A Guide to Machine Learning Engineering","author":"G Hulten","year":"2018","unstructured":"Hulten, G.: Building Intelligent Systems: A Guide to Machine Learning Engineering, 1st edn. Apress, New York (2018)","edition":"1"},{"key":"6_CR21","doi-asserted-by":"crossref","unstructured":"K\u00e4stner, C., Kang, E.: Teaching Software Engineering for AI-Enabled Systems. arXiv:2001.06691 [cs], January 2020","DOI":"10.1145\/3377814.3381714"},{"key":"6_CR22","doi-asserted-by":"crossref","unstructured":"Serban, A., van der Blom, K., Hoos, H., Visser, J.: Adoption and effects of software engineering best practices in machine learning. In: Proceedings of the 14th International Symposium on Empirical Software Engineering and Measurement (2020)","DOI":"10.1145\/3382494.3410681"},{"key":"6_CR23","doi-asserted-by":"crossref","unstructured":"Bosch, J., Crnkovic, I., Olsson, H.H.: Engineering AI Systems: A Research Agenda. arXiv:2001.07522 [cs], January 2020","DOI":"10.4018\/978-1-7998-5101-1.ch001"},{"key":"6_CR24","unstructured":"Zhang, J.M., et al.: Machine learning testing: survey, landscapes and horizons. IEEE Trans. Softw. Eng. (2020). (Early Access)"},{"key":"6_CR25","doi-asserted-by":"publisher","first-page":"5193","DOI":"10.1007\/s10664-020-09881-0","volume":"25","author":"R Vincenzo","year":"2020","unstructured":"Vincenzo, R., Jahangirova, G., Stocco, A., Humbatova, N., Weiss, M., Tonella, P.: Testing machine learning based systems: a systematic mapping. Empirical Softw. Eng. 25, 5193\u20135254 (2020)","journal-title":"Empirical Softw. Eng."},{"key":"6_CR26","series-title":"SpringerBriefs in Business","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1007\/978-3-319-72844-5_2","volume-title":"Digital Transformation Now!","author":"DRA Schallmo","year":"2018","unstructured":"Schallmo, D.R.A., Williams, C.A.: History of digital transformation. Digital Transformation Now!. SB, pp. 3\u20138. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-319-72844-5_2"},{"key":"6_CR27","volume-title":"Artificial Intelligence: A Modern Approach","author":"S Russell","year":"2009","unstructured":"Russell, S., Norvig, P.: Artificial Intelligence: A Modern Approach, 3rd edn. Pearson, Upper Saddle River (2009)","edition":"3"},{"issue":"14","key":"6_CR28","doi-asserted-by":"publisher","first-page":"841","DOI":"10.1016\/S0950-5849(02)00108-8","volume":"44","author":"KY Cai","year":"2002","unstructured":"Cai, K.Y.: Optimal software testing and adaptive software testing in the context of software cybernetics. Inf. Softw. Technol. 44(14), 841\u2013855 (2002)","journal-title":"Inf. Softw. Technol."},{"key":"6_CR29","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.infsof.2017.03.013","volume":"90","author":"S Mahdavi-Hezavehi","year":"2017","unstructured":"Mahdavi-Hezavehi, S., et al.: A systematic literature review on methods that handle multiple quality attributes in architecture-based self-adaptive systems. Inf. Softw. Technol. 90, 1\u201326 (2017)","journal-title":"Inf. Softw. Technol."},{"key":"6_CR30","unstructured":"Sculley, D., et al.: Hidden technical debt in machine learning systems. In: Proceedings of the 28th International Conference on Neural Information Processing Systems, pp. 2503\u20132511 (2015)"},{"key":"6_CR31","doi-asserted-by":"crossref","unstructured":"Humbatova, N., Jahangirova, G., Bavota, G., Riccio, V., Stocco, A., Tonella, P.: Taxonomy of real faults in deep learning systems. In: Proceedings of the 42nd International Conference on Software Engineering (2020)","DOI":"10.1145\/3377811.3380395"},{"key":"6_CR32","doi-asserted-by":"publisher","DOI":"10.1017\/9781316771273","volume-title":"Introduction to Software Testing","author":"P Ammann","year":"2016","unstructured":"Ammann, P., Offutt, J.: Introduction to Software Testing. Cambridge University Press, Cambridge (2016)"},{"key":"6_CR33","doi-asserted-by":"publisher","first-page":"129","DOI":"10.1007\/978-3-030-25312-7_6","volume-title":"Security and Quality in Cyber-Physical Systems Engineering","author":"M Felderer","year":"2019","unstructured":"Felderer, M., Russo, B., Auer, F.: On testing data-intensive software systems. Security and Quality in Cyber-Physical Systems Engineering, pp. 129\u2013148. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-25312-7_6"},{"issue":"12","key":"6_CR34","doi-asserted-by":"publisher","first-page":"1278","DOI":"10.1109\/TSE.1987.232881","volume":"SE\u201313","author":"V Basili","year":"1987","unstructured":"Basili, V., Selby, R.: Comparing the effectiveness of software testing strategies. IEEE Trans. Softw. Eng. SE\u201313(12), 1278\u20131296 (1987)","journal-title":"IEEE Trans. Softw. Eng."},{"issue":"6","key":"6_CR35","doi-asserted-by":"publisher","first-page":"e1675","DOI":"10.1002\/stvr.1675","volume":"28","author":"Q Zhu","year":"2018","unstructured":"Zhu, Q., Panichella, A., Zaidman, A.: A systematic literature review of how mutation testing supports quality assurance processes. Softw. Test. Verif. Reliab. 28(6), e1675 (2018)","journal-title":"Softw. Test. Verif. Reliab."},{"issue":"6","key":"6_CR36","doi-asserted-by":"publisher","first-page":"e1885","DOI":"10.1002\/smr.1885","volume":"29","author":"F Erich","year":"2017","unstructured":"Erich, F., Amrit, C., Daneva, M.: A qualitative study of DevOps usage in practice. J. Softw. Evol. Process 29(6), e1885 (2017)","journal-title":"J. Softw. Evol. Process"},{"issue":"7","key":"6_CR37","doi-asserted-by":"publisher","first-page":"363","DOI":"10.3390\/info11070363","volume":"11","author":"I Karamitsos","year":"2020","unstructured":"Karamitsos, I., Albarhami, S., Apostolopoulos, C.: Applying DevOps practices of continuous automation for machine learning. Information 11(7), 363 (2020)","journal-title":"Information"}],"container-title":["Lecture Notes in Business Information Processing","Software Quality: Future Perspectives on Software Engineering Quality"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-65854-0_6","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,7]],"date-time":"2024-03-07T15:20:26Z","timestamp":1709824826000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-65854-0_6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030658533","9783030658540"],"references-count":37,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-65854-0_6","relation":{},"ISSN":["1865-1348","1865-1356"],"issn-type":[{"value":"1865-1348","type":"print"},{"value":"1865-1356","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"6 January 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"SWQD","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Software Quality","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Vienna","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Austria","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"19 January 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"21 January 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"13","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"swqd2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.software-quality-days.com\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"13","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"5","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"23% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4-5","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"1-2","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"1 paper was desk rejected; furthermore, 2 invited keynote papers and 1 invited introductory paper for an interactive session was published. The conference did not take place due to the COVID-19 pandemic and will be merged with SWQD 2022.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}