{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,12]],"date-time":"2026-07-12T03:16:55Z","timestamp":1783826215361,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":58,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T00:00:00Z","timestamp":1726012800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,9,11]]},"DOI":"10.1145\/3650212.3680312","type":"proceedings-article","created":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T11:44:25Z","timestamp":1726055065000},"page":"678-690","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["Test Selection for Deep Neural Networks using Meta-Models with Uncertainty Metrics"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7497-5957","authenticated-orcid":false,"given":"Demet","family":"Demir","sequence":"first","affiliation":[{"name":"Graduate School of Informatics, Middle East Technical University, Ankara, T\u00fcrkiye"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4828-0190","authenticated-orcid":false,"given":"Aysu","family":"Betin Can","sequence":"additional","affiliation":[{"name":"Graduate School of Informatics, Middle East Technical University, Ankara, T\u00fcrkiye"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0738-6669","authenticated-orcid":false,"given":"Elif","family":"Surer","sequence":"additional","affiliation":[{"name":"Graduate School of Informatics, Middle East Technical University, Ankara, T\u00fcrkiye"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,9,11]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSE51541.2021.9582499"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1109\/AITest.2019.000-6"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/AITEST49225.2020.00010"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1088\/2632-2153"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2017.49"},{"key":"e_1_3_2_1_6_1","volume-title":"Proceedings of the Twenty-Second International Conference on Artificial Intelligence and Statistics. 89","author":"Chen Tongfei","year":"2019","unstructured":"Tongfei Chen, Jiri Navratil, Vijay Iyengar, and Karthikeyan Shanmugam. 2019. Confidence Scoring Using Whitebox Meta-models with Linear Classifier Probes. In Proceedings of the Twenty-Second International Conference on Artificial Intelligence and Statistics. 89, PMLR, 1467\u20131475."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","unstructured":"Tarin Clanuwat Mikel Bober-Irizar Asanobu Kitamoto Alex Lamb Kazuaki Yamamoto and David Ha. 2018. Deep learning for classical japanese literature. arXiv preprint arXiv:1812.01718 https:\/\/doi.org\/10.48550\/arXiv.1812.01718 10.48550\/arXiv.1812.01718","DOI":"10.48550\/arXiv.1812.01718"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2023.3327820"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00032"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","unstructured":"Alexey Dosovitskiy Lucas Beyer Alexander Kolesnikov Dirk Weissenborn Xiaohua Zhai Thomas Unterthiner Mostafa Dehghani Matthias Minderer Georg Heigold Sylvain Gelly Jacob Uszkoreit and Neil Houlsby. 2020. An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale. arXiv preprint arXiv:2010.11929 https:\/\/doi.org\/10.48550\/arXiv.2010.11929 10.48550\/arXiv.2010.11929","DOI":"10.48550\/arXiv.2010.11929"},{"key":"e_1_3_2_1_12_1","volume-title":"Software Quality: Future Perspectives on Software Engineering Quality","author":"Felderer Michael","unstructured":"Michael Felderer and Rudolf Ramler. 2021. Quality Assurance for AI-Based Systems: Overview and Challenges (Introduction to Interactive Session). In Software Quality: Future Perspectives on Software Engineering Quality. Springer International Publishing, 33\u201342. isbn:978-3-030-65854-0"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3395363.3397357"},{"key":"e_1_3_2_1_14_1","first-page":"7068","article-title":"Exploring the Limits of Out-of-Distribution Detection","author":"Fort Stanislav","year":"2021","unstructured":"Stanislav Fort, Jie Ren, and Balaji Lakshminarayanan. 2021. Exploring the Limits of Out-of-Distribution Detection. In Advances in Neural Information Processing Systems. 34, Curran Associates, Inc., 7068\u20137081.","journal-title":"Advances in Neural Information Processing Systems. 34, Curran Associates, Inc."},{"key":"e_1_3_2_1_15_1","volume-title":"Proceedings of the 33rd International Conference on International Conference on Machine Learning-Volume 48","author":"Gal Yarin","year":"2016","unstructured":"Yarin Gal and Zoubin Ghahramani. 2016. Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning. In Proceedings of the 33rd International Conference on International Conference on Machine Learning-Volume 48. 48, PMLR, 1050\u20131059."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3422622"},{"key":"e_1_3_2_1_17_1","volume-title":"Proceedings of the International Conference on Learning Representations. arxiv:1412","author":"Goodfellow Ian J","year":"2015","unstructured":"Ian J Goodfellow, Jonathon Shlens, and Christian Szegedy. 2015. Explaining and harnessing adversarial examples. In Proceedings of the International Conference on Learning Representations. arxiv:1412.6572"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3264835"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409754"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_1_21_1","volume-title":"Proceedings of the 39th International Conference on Machine Learning. 162","author":"Hou Liang","year":"2022","unstructured":"Liang Hou, Qi Cao, Huawei Shen, Siyuan Pan, Xiaoshuang Li, and Xueqi Cheng. 2022. Conditional GANs with Auxiliary Discriminative Classifier. In Proceedings of the 39th International Conference on Machine Learning. 162, PMLR, 8888\u20138902."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511598"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3387940.3391456"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3546947"},{"key":"e_1_3_2_1_26_1","volume-title":"Learning multiple layers of features from tiny images","author":"Krizhevsky Alex","unstructured":"Alex Krizhevsky and Geoffrey Hinton. 2009. Learning multiple layers of features from tiny images. University of Toronto."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.5555\/3295222.3295387"},{"key":"e_1_3_2_1_28_1","unstructured":"Yann LeCun. 1998. The MNIST database of handwritten digits. http:\/\/yann.lecun.com\/exdb\/mnist\/"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/5.726791"},{"key":"e_1_3_2_1_30_1","first-page":"20874","article-title":"TestRank: Bringing Order into Unlabeled Test Instances for Deep Learning Tasks","author":"YU LI","year":"2021","unstructured":"YU LI, Min LI, Qiuxia LAI, Yannan Liu, and Qiang Xu. 2021. TestRank: Bringing Order into Unlabeled Test Instances for Deep Learning Tasks. In Advances in Neural Information Processing Systems. 34, Curran Associates, Inc., 20874\u201320886.","journal-title":"Advances in Neural Information Processing Systems. 34, Curran Associates, Inc."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-NIER.2019.00031"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3417330"},{"key":"e_1_3_2_1_34_1","volume-title":"Proceedings of the International Conference on Learning Representations.","author":"Madry Aleksander","year":"2018","unstructured":"Aleksander Madry, Aleksandar Makelov, Ludwig Schmidt, Dimitris Tsipras, and Adrian Vladu. 2018. Towards deep learning models resistant to adversarial attacks. In Proceedings of the International Conference on Learning Representations."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/SEAA56994.2022.00020"},{"key":"e_1_3_2_1_36_1","unstructured":"Jiri Navratil Matthew Arnold and Benjamin Elder. 2020. Uncertainty prediction for deep sequential regression using meta models. arXiv preprint arXiv:2007.01350."},{"key":"e_1_3_2_1_37_1","volume-title":"Proceedings of the NIPS Workshop on Deep Learning and Unsupervised Feature Learning.","author":"Netzer Yuval","year":"2011","unstructured":"Yuval Netzer, Tao Wang, Adam Coates, Alessandro Bissacco, Bo Wu, and Andrew Y Ng. 2011. Reading digits in natural images with unsupervised feature learning. In Proceedings of the NIPS Workshop on Deep Learning and Unsupervised Feature Learning."},{"key":"e_1_3_2_1_38_1","volume-title":"Beat Buesser, Ambrish Rawat, Martin Wistuba, Valentina Zantedeschi, Nathalie Baracaldo, Bryant Chen, Heiko Ludwig, Ian Molloy, and Ben Edwards.","author":"Nicolae Maria-Irina","year":"2019","unstructured":"Maria-Irina Nicolae, Mathieu Sinn, Minh Ngoc Tran, Beat Buesser, Ambrish Rawat, Martin Wistuba, Valentina Zantedeschi, Nathalie Baracaldo, Bryant Chen, Heiko Ludwig, Ian Molloy, and Ben Edwards. 2019. Adversarial Robustness Toolbox v1.0.0. arxiv:1807.01069. arxiv:1807.01069"},{"key":"e_1_3_2_1_39_1","volume-title":"Proceedings of the 36th International Conference on Machine Learning (Proceedings of Machine Learning Research","volume":"4911","author":"Odena Augustus","year":"2019","unstructured":"Augustus Odena, Catherine Olsson, David Andersen, and Ian Goodfellow. 2019. TensorFuzz: Debugging Neural Networks with Coverage-Guided Fuzzing. In Proceedings of the 36th International Conference on Machine Learning (Proceedings of Machine Learning Research, Vol. 97). PMLR, 4901\u20134911."},{"key":"e_1_3_2_1_40_1","volume-title":"Advances in Neural Information Processing Systems. 32, Curran Associates","author":"Ovadia Yaniv","unstructured":"Yaniv Ovadia, Emily Fertig, Jie Ren, Zachary Nado, D. Sculley, Sebastian Nowozin, Joshua Dillon, Balaji Lakshminarayanan, and Jasper Snoek. 2019. Can you trust your model' s uncertainty? Evaluating predictive uncertainty under dataset shift. In Advances in Neural Information Processing Systems. 32, Curran Associates, Inc.."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-020-09881-0"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v37i8.26167"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416621"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1109\/QRS54544.2021.00027"},{"key":"e_1_3_2_1_46_1","volume-title":"Proceedings of the International Conference on Learning Representations. 1\u201314","author":"Simonyan Karen","year":"2015","unstructured":"Karen Simonyan and Andrew Zisserman. 2015. Very deep convolutional networks for large-scale image recognition. In Proceedings of the International Conference on Learning Representations. 1\u201314."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-Companion.2019.00051"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-22064-7_27"},{"key":"e_1_3_2_1_49_1","unstructured":"Begum Taskazan Jiri Navratil Matthew Arnold Anupama Murthi Ganesh Venkataraman and Benjamin Elder. 2020. Not your grandfathers test set: Reducing labeling effort for testing. arXiv preprint arXiv:2007.05499."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00046"},{"key":"e_1_3_2_1_52_1","unstructured":"Zhendong Wang Huangjie Zheng Pengcheng He Weizhu Chen and Mingyuan Zhou. 2023. Diffusion-GAN: Training GANs with Diffusion. arXiv preprint arXiv:2206.02262."},{"key":"e_1_3_2_1_53_1","unstructured":"Han Xiao Kashif Rasul and Roland Vollgraf. 2017. Fashion-mnist: a novel image dataset for benchmarking machine learning algorithms. arXiv preprint arXiv:1708.07747."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330579"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.scico.2021.102761"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2962027"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380422"}],"event":{"name":"ISSTA '24: 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis","location":"Vienna Austria","acronym":"ISSTA '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","AITO"]},"container-title":["Proceedings of the 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650212.3680312","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3650212.3680312","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T22:50:07Z","timestamp":1750287007000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650212.3680312"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,11]]},"references-count":58,"alternative-id":["10.1145\/3650212.3680312","10.1145\/3650212"],"URL":"https:\/\/doi.org\/10.1145\/3650212.3680312","relation":{},"subject":[],"published":{"date-parts":[[2024,9,11]]},"assertion":[{"value":"2024-09-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}