{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,13]],"date-time":"2026-05-13T11:33:35Z","timestamp":1778672015860,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":47,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,5,16]],"date-time":"2022-05-16T00:00:00Z","timestamp":1652659200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Kompetensfonden"},{"name":"Wallenberg AI, Autonomous Systems and Software Program (WASP)"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,5,16]]},"DOI":"10.1145\/3522664.3528596","type":"proceedings-article","created":{"date-parts":[[2022,10,17]],"date-time":"2022-10-17T16:30:14Z","timestamp":1666024214000},"page":"10-21","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["Exploring ML testing in practice"],"prefix":"10.1145","author":[{"given":"Qunying","family":"Song","sequence":"first","affiliation":[{"name":"Lund University, Lund, Sweden"}]},{"given":"Markus","family":"Borg","sequence":"additional","affiliation":[{"name":"Lund University, Lund, Sweden"}]},{"given":"Emelie","family":"Engstr\u00f6m","sequence":"additional","affiliation":[{"name":"Lund University, Lund, Sweden"}]},{"given":"H\u00e5kan","family":"Ard\u00f6","sequence":"additional","affiliation":[{"name":"Axis Communications, Lund, Sweden"}]},{"given":"Sergio","family":"Rico","sequence":"additional","affiliation":[{"name":"Lund University, Lund, Sweden"}]}],"member":"320","published-online":{"date-parts":[[2022,10,17]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-018-9670-1"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"e_1_3_2_1_3_1","volume-title":"Challenges and practices in aligning requirements with verification and validation: a case study of six companies. Empirical software engineering 19, 6","author":"Bjarnason Elizabeth","year":"2014","unstructured":"Elizabeth Bjarnason, Per Runeson, Markus Borg, Michael Unterkalmsteiner, Emelie Engstrom, Bjorn Regnell, Giedre Sabaliauskaite, Annabella Loconsole, Tony Gorschek, and Robert Feldt. 2014. Challenges and practices in aligning requirements with verification and validation: a case study of six companies. Empirical software engineering 19, 6 (2014), 1809--1855."},{"key":"e_1_3_2_1_4_1","volume-title":"Towards corner case detection for autonomous driving. In 2019 IEEE Intelligent vehicles symposium (IV)","author":"Bolte Jan-Aike","unstructured":"Jan-Aike Bolte, Andreas Bar, Daniel Lipinski, and Tim Fingscheidt. 2019. Towards corner case detection for autonomous driving. In 2019 IEEE Intelligent vehicles symposium (IV). IEEE, 438--445."},{"key":"e_1_3_2_1_5_1","volume-title":"Software Quality: Future Perspectives on Software Engineering Quality","author":"Borg Markus","unstructured":"Markus Borg. 2021. The AIQ Meta-Testbed: Pragmatically Bridging Academic AI Testing and Industrial Q Needs. In Software Quality: Future Perspectives on Software Engineering Quality, Dietmar Winkler, Stefan Biffl, Daniel Mendez, Manuel Wimmer, and Johannes Bergsmann (Eds.). Springer International Publishing, Cham, 66--77."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-94238-0_1"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"crossref","unstructured":"Markus Borg Joshua Bronson Linus Christensson Fredrik Olsson Olof Lennartsson Elias Sonnsj\u00f6 Hamid Ebadi and Martin Karsberg. 2021. Exploring the Assessment List for Trustworthy AI in the Context of Advanced Driver-Assistance Systems. In 2021 IEEE\/ACM 2nd International Workshop on Ethics in Software Engineering Research and Practice (SEthics). IEEE 5--12.","DOI":"10.1109\/SEthics52569.2021.00009"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.2991\/jase.d.190131.001"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW52544.2021.00039"},{"key":"e_1_3_2_1_10_1","volume-title":"Helena Holmstr\u00f6m Olsson, and Ivica Crnkovic","author":"Bosch Jan","year":"2021","unstructured":"Jan Bosch, Helena Holmstr\u00f6m Olsson, and Ivica Crnkovic. 2021. Engineering ai systems: A research agenda. In Artificial Intelligence Paradigms for Smart Cyber-Physical Systems. IGI Global, 1--19."},{"key":"e_1_3_2_1_11_1","unstructured":"Eric Breck Neoklis Polyzotis Sudip Roy Steven Whang and Martin Zinkevich. 2019. Data Validation for Machine Learning. In MLSys."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/AITest.2019.000-6"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2020.2987673"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-32489-6_13"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/MEMCOD.2018.8556962"},{"key":"e_1_3_2_1_16_1","volume-title":"Markus Borg, Gregory Gay, Afonso Fontes, and Kasper Socha.","author":"Ebadi Hamid","year":"2021","unstructured":"Hamid Ebadi, Mahshid Helali Moghadam, Markus Borg, Gregory Gay, Afonso Fontes, and Kasper Socha. 2021. Efficient and Effective Generation of Test Cases for Pedestrian Detection-Search-based Software Testing of Baidu Apollo in SVL. In 2021 IEEE International Conference on Artificial Intelligence Testing (AITest). IEEE, 103--110."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11219-016-9322-x"},{"key":"e_1_3_2_1_18_1","volume-title":"Security and Quality in Cyber-Physical Systems Engineering","author":"Felderer Michael","unstructured":"Michael Felderer, Barbara Russo, and Florian Auer. 2019. On testing data-intensive software systems. In Security and Quality in Cyber-Physical Systems Engineering. Springer, 129--148."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/AITest.2019.00-12"},{"key":"e_1_3_2_1_20_1","volume-title":"NIPS MLSys Workshop.","author":"Hynes Nick","year":"2017","unstructured":"Nick Hynes, D Sculley, and Michael Terry. 2017. The data linter: Lightweight, automated sanity checking for ml data sets. In NIPS MLSys Workshop."},{"key":"e_1_3_2_1_21_1","unstructured":"ISO\/IEC. 2008. ISO 25012 Systems and software engineering - Systems and software quality requirements and evaluation (SQuaRE) - Data quality model."},{"key":"e_1_3_2_1_22_1","unstructured":"ISO\/IEC. 2011. ISO 25010 Systems and software engineering - Systems and software quality requirements and evaluation (SQuaRE) - System and software quality models."},{"key":"e_1_3_2_1_23_1","volume-title":"Artificial intelligence in healthcare: past, present and future. Stroke and vascular neurology 2, 4","author":"Jiang Fei","year":"2017","unstructured":"Fei Jiang, Yong Jiang, Hui Zhi, Yi Dong, Hao Li, Sufeng Ma, Yilong Wang, Qiang Dong, Haipeng Shen, and Yongjun Wang. 2017. Artificial intelligence in healthcare: past, present and future. Stroke and vascular neurology 2, 4 (2017)."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3417065"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.14778\/2994509.2994514"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2020.106368"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2018.00021"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/AITest.2019.00010"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/SBST52555.2021.00018"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2647648.2647650"},{"key":"e_1_3_2_1_32_1","volume-title":"Lin (Eds.)","volume":"33","author":"Pleiss Geoff","year":"2020","unstructured":"Geoff Pleiss, Tianyi Zhang, Ethan Elenberg, and Kilian Q Weinberger. 2020. Identifying Mislabeled Data using the Area Under the Margin Ranking. In Advances in Neural Information Processing Systems, H. Larochelle, M. Ranzato, R. Hadsell, M. F. Balcan, and H. Lin (Eds.), Vol. 33. 17044--17056."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-020-09881-0"},{"key":"e_1_3_2_1_34_1","unstructured":"Sergio Rico N. Ali Emelie Engstr\u00f6m and Martin H\u00f6st. 2020. Guidelines for conducting interactive rapid reviews in software engineering - from a focus on technology transfer to knowledge exchange."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.14778\/3229863.3229867"},{"key":"e_1_3_2_1_36_1","volume-title":"Proc. of the 28th Int'l Conf. on Neural Information Proc. Systems. 2503--2511","author":"D. Sculley","year":"2015","unstructured":"D. Sculley et al. 2015. Hidden Technical Debt in Machine Learning Systems. In Proc. of the 28th Int'l Conf. on Neural Information Proc. Systems. 2503--2511."},{"key":"e_1_3_2_1_37_1","volume-title":"Muhammad Uzair khan, and Muhammad Zohaib Iqbal","author":"Sherin Salman","year":"2019","unstructured":"Salman Sherin, Muhammad Uzair khan, and Muhammad Zohaib Iqbal. 2019. A Systematic Mapping Study on Testing of Machine Learning Programs. arXiv:cs.LG\/1907.09427"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","unstructured":"Qunying Song Markus Borg Emelie Engstr\u00f6m H\u00e5kan Ard\u00f6 and Sergio Rico. 2022. Primary Studies. 10.5281\/zenodo.5865070","DOI":"10.5281\/zenodo.5865070"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/ESEM.2017.28"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jclinepi.2020.06.029"},{"key":"e_1_3_2_1_42_1","unstructured":"Jonathan Uesato Ananya Kumar Csaba Szepesvari Tom Erez Avraham Ruderman Keith Anderson Nicolas Heess Pushmeet Kohli et al. 2018. Rigorous agent evaluation: An adversarial approach to uncover catastrophic failures. arXiv preprint arXiv: 1812.01647 (2018)."},{"key":"e_1_3_2_1_43_1","volume-title":"Certification of embedded systems based on Machine Learning: A survey. arXiv preprint arXiv.2106.07221","author":"Vidot Guillaume","year":"2021","unstructured":"Guillaume Vidot, Christophe Gabreau, Ileana Ober, and Iulian Ober. 2021. Certification of embedded systems based on Machine Learning: A survey. arXiv preprint arXiv.2106.07221 (2021)."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1109\/REW.2019.00050"},{"key":"e_1_3_2_1_45_1","unstructured":"Jie Zhang Earl Barr Benjamin Guedj Mark Harman and John Shawe-Taylor. 2019. Perturbed model validation: A new framework to validate model relevance. (2019)."},{"key":"e_1_3_2_1_46_1","volume-title":"Machine learning testing: Survey, landscapes and horizons","author":"Zhang Jie M","year":"2020","unstructured":"Jie M Zhang, Mark Harman, Lei Ma, and Yang Liu. 2020. Machine learning testing: Survey, landscapes and horizons. IEEE Transactions on Software Engineering (2020)."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"}],"event":{"name":"CAIN '22: 1st Conference on AI Engineering - Software Engineering for AI","location":"Pittsburgh Pennsylvania","acronym":"CAIN '22","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE TCSC IEEE Technical Committee on Scalable Computing"]},"container-title":["Proceedings of the 1st International Conference on AI Engineering: Software Engineering for AI"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3522664.3528596","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3522664.3528596","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:09:34Z","timestamp":1750183774000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3522664.3528596"}},"subtitle":["lessons learned from an interactive rapid review with axis communications"],"short-title":[],"issued":{"date-parts":[[2022,5,16]]},"references-count":47,"alternative-id":["10.1145\/3522664.3528596","10.1145\/3522664"],"URL":"https:\/\/doi.org\/10.1145\/3522664.3528596","relation":{},"subject":[],"published":{"date-parts":[[2022,5,16]]},"assertion":[{"value":"2022-10-17","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}