{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,24]],"date-time":"2026-04-24T14:55:55Z","timestamp":1777042555703,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":46,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,7,18]],"date-time":"2022-07-18T00:00:00Z","timestamp":1658102400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,7,18]]},"DOI":"10.1145\/3533767.3534375","type":"proceedings-article","created":{"date-parts":[[2022,7,15]],"date-time":"2022-07-15T14:28:50Z","timestamp":1657895330000},"page":"139-150","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":52,"title":["Simple techniques work surprisingly well for neural network test prioritization and active learning (replicability study)"],"prefix":"10.1145","author":[{"given":"Michael","family":"Weiss","sequence":"first","affiliation":[{"name":"USI Lugano, Switzerland"}]},{"given":"Paolo","family":"Tonella","sequence":"additional","affiliation":[{"name":"USI Lugano, Switzerland"}]}],"member":"320","published-online":{"date-parts":[[2022,7,18]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"[n. d.]. ACM Artifact Review and Badging \u2013 Version 2.0. https:\/\/www.acm.org\/publications\/policies\/artifact-review-badging Accessed: 2021-08-12 \t\t\t\t\t  [n. d.]. ACM Artifact Review and Badging \u2013 Version 2.0. https:\/\/www.acm.org\/publications\/policies\/artifact-review-badging Accessed: 2021-08-12"},{"key":"e_1_3_2_1_2_1","unstructured":"[n. d.]. ReScience C What\u2019s the difference between replication and reproduction? http:\/\/rescience.github.io\/faq\/ Accessed: 2021-08-12 \t\t\t\t\t  [n. d.]. ReScience C What\u2019s the difference between replication and reproduction? http:\/\/rescience.github.io\/faq\/ Accessed: 2021-08-12"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/CLUSTR.2009.5289161"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/issrew51248.2020.00073"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/32.962562"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-90-481-8847-5_10"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3395363.3397357"},{"key":"e_1_3_2_1_8_1","volume-title":"Uncertainty in Deep Learning. Ph. D. Dissertation","author":"Gal Yarin","unstructured":"Yarin Gal . 2016. Uncertainty in Deep Learning. Ph. D. Dissertation . University of Cambridge . Yarin Gal. 2016. Uncertainty in Deep Learning. Ph. D. Dissertation. University of Cambridge."},{"key":"e_1_3_2_1_9_1","volume-title":"Proceedings of the 33rd International Conference on International Conference on Machine Learning -","volume":"48","author":"Gal Yarin","year":"2016","unstructured":"Yarin Gal and Zoubin Ghahramani . 2016 . Dropout As a Bayesian Approximation: Representing Model Uncertainty in Deep Learning . In Proceedings of the 33rd International Conference on International Conference on Machine Learning - Volume 48 (ICML\u201916). JMLR.org, 1050\u20131059. http:\/\/dl.acm.org\/citation.cfm?id=3045390.3045502 Yarin Gal and Zoubin Ghahramani. 2016. Dropout As a Bayesian Approximation: Representing Model Uncertainty in Deep Learning. In Proceedings of the 33rd International Conference on International Conference on Machine Learning - Volume 48 (ICML\u201916). JMLR.org, 1050\u20131059. http:\/\/dl.acm.org\/citation.cfm?id=3045390.3045502"},{"key":"e_1_3_2_1_10_1","volume-title":"International Conference on Machine Learning. 1183\u20131192","author":"Gal Yarin","year":"2017","unstructured":"Yarin Gal , Riashat Islam , and Zoubin Ghahramani . 2017 . Deep bayesian active learning with image data . In International Conference on Machine Learning. 1183\u20131192 . Yarin Gal, Riashat Islam, and Zoubin Ghahramani. 2017. Deep bayesian active learning with image data. In International Conference on Machine Learning. 1183\u20131192."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3221269.3223036"},{"key":"e_1_3_2_1_12_1","volume-title":"International Conference on Learning Representations, arXiv:1903","author":"Hendrycks Dan","year":"2018","unstructured":"Dan Hendrycks and Thomas Dietterich . 2018 . Benchmarking Neural Network Robustness to Common Corruptions and Perturbations . International Conference on Learning Representations, arXiv:1903 .12261v1. Dan Hendrycks and Thomas Dietterich. 2018. Benchmarking Neural Network Robustness to Common Corruptions and Perturbations. International Conference on Learning Representations, arXiv:1903.12261v1."},{"key":"e_1_3_2_1_13_1","unstructured":"Dan Hendrycks and Kevin Gimpel. 2016. A Baseline for Detecting Misclassified and Out-of-Distribution Examples in Neural Networks. arXiv:1610.02136v3. \t\t\t\t\t  Dan Hendrycks and Kevin Gimpel. 2016. A Baseline for Detecting Misclassified and Out-of-Distribution Examples in Neural Networks. arXiv:1610.02136v3."},{"key":"e_1_3_2_1_14_1","volume-title":"CVPR Workshop on Autonomous Driving - Andrej Karpathy details Tesla\u2019s Self Driving Car Strategy. https:\/\/youtu.be\/gZ2SeiLjaEc","author":"Karpathy Andrej","unstructured":"Andrej Karpathy . [n. d.]. CVPR Workshop on Autonomous Driving - Andrej Karpathy details Tesla\u2019s Self Driving Car Strategy. https:\/\/youtu.be\/gZ2SeiLjaEc Andrej Karpathy. [n. d.]. CVPR Workshop on Autonomous Driving - Andrej Karpathy details Tesla\u2019s Self Driving Car Strategy. https:\/\/youtu.be\/gZ2SeiLjaEc"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/icse.2019.00108"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3417065"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/AST52587.2021.00017"},{"key":"e_1_3_2_1_18_1","unstructured":"Alex Krizhevsky. 2009. Learning multiple layers of features from tiny images. \t\t\t\t\t  Alex Krizhevsky. 2009. Learning multiple layers of features from tiny images."},{"key":"e_1_3_2_1_19_1","volume-title":"MNIST handwritten digit database. ATT Labs [Online]. Available: http:\/\/yann.lecun.com\/exdb\/mnist, 2","author":"LeCun Yann","year":"2010","unstructured":"Yann LeCun , Corinna Cortes , and CJ Burges . 2010. MNIST handwritten digit database. ATT Labs [Online]. Available: http:\/\/yann.lecun.com\/exdb\/mnist, 2 ( 2010 ). Yann LeCun, Corinna Cortes, and CJ Burges. 2010. MNIST handwritten digit database. ATT Labs [Online]. Available: http:\/\/yann.lecun.com\/exdb\/mnist, 2 (2010)."},{"key":"e_1_3_2_1_20_1","unstructured":"Vladimir I Levenshtein. 1966. Binary codes capable of correcting deletions insertions and reversals. In Soviet physics doklady. 10 707\u2013710. \t\t\t\t\t  Vladimir I Levenshtein. 1966. Binary codes capable of correcting deletions insertions and reversals. In Soviet physics doklady. 10 707\u2013710."},{"key":"e_1_3_2_1_21_1","volume-title":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing: System Demonstrations. Association for Computational Linguistics, 175\u2013184","author":"Lhoest Quentin","year":"2021","unstructured":"Quentin Lhoest , Albert Villanova del Moral , Patrick von Platen , Thomas Wolf , Mario \u0160a\u0161ko , Yacine Jernite , Abhishek Thakur , Lewis Tunstall , Suraj Patil , Mariama Drame , Julien Chaumond , Julien Plu , Joe Davison , Simon Brandeis , Victor Sanh , Teven Le Scao , Kevin Canwen Xu , Nicolas Patry , Steven Liu , Angelina McMillan-Major , Philipp Schmid , Sylvain Gugger , Nathan Raw , Sylvain Lesage , Anton Lozhkov , Matthew Carrigan , Th\u00e9o Matussi\u00e8re , Leandro von Werra , Lysandre Debut , Stas Bekman , and Cl\u00e9ment Delangue . 2021 . Datasets: A Community Library for Natural Language Processing . In Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing: System Demonstrations. Association for Computational Linguistics, 175\u2013184 . https:\/\/aclanthology.org\/2021.emnlp-demo.21 Quentin Lhoest, Albert Villanova del Moral, Patrick von Platen, Thomas Wolf, Mario \u0160a\u0161ko, Yacine Jernite, Abhishek Thakur, Lewis Tunstall, Suraj Patil, Mariama Drame, Julien Chaumond, Julien Plu, Joe Davison, Simon Brandeis, Victor Sanh, Teven Le Scao, Kevin Canwen Xu, Nicolas Patry, Steven Liu, Angelina McMillan-Major, Philipp Schmid, Sylvain Gugger, Nathan Raw, Sylvain Lesage, Anton Lozhkov, Matthew Carrigan, Th\u00e9o Matussi\u00e8re, Leandro von Werra, Lysandre Debut, Stas Bekman, and Cl\u00e9ment Delangue. 2021. Datasets: A Community Library for Natural Language Processing. In Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing: System Demonstrations. Association for Computational Linguistics, 175\u2013184. https:\/\/aclanthology.org\/2021.emnlp-demo.21"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180213"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3417330"},{"key":"e_1_3_2_1_25_1","unstructured":"Prasanta Chandra Mahalanobis. 1936. On the generalized distance in statistics. \t\t\t\t\t  Prasanta Chandra Mahalanobis. 1936. On the generalized distance in statistics."},{"key":"e_1_3_2_1_26_1","unstructured":"Norman Mu and Justin Gilmer. 2019. MNIST-C: A Robustness Benchmark for Computer Vision. CoRR arXiv:1906.02337v1. \t\t\t\t\t  Norman Mu and Justin Gilmer. 2019. MNIST-C: A Robustness Benchmark for Computer Vision. CoRR arXiv:1906.02337v1."},{"key":"e_1_3_2_1_27_1","volume-title":"CS299 - Deep Learning","author":"Andrew Ng.","unstructured":"Andrew Ng. [n. d.]. Virtuous Circle of AI. Lecture Slides , CS299 - Deep Learning , Stanford University . https:\/\/cs229.stanford.edu\/materials\/CS229-DeepLearning.pdf Accessed: 2021-13-12 Andrew Ng. [n. d.]. Virtuous Circle of AI. Lecture Slides, CS299 - Deep Learning, Stanford University. https:\/\/cs229.stanford.edu\/materials\/CS229-DeepLearning.pdf Accessed: 2021-13-12"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/cvpr.2015.7298640"},{"key":"e_1_3_2_1_29_1","unstructured":"Yaniv Ovadia Emily Fertig Jie Ren Zachary Nado D. Sculley Sebastian Nowozin Joshua Dillon Balaji Lakshminarayanan and Jasper Snoek. 2019. Can you trust your models uncertainty? Evaluating predictive uncertainty under dataset shift. Advances in Neural Information Processing Systems 13991\u201314002. \t\t\t\t\t  Yaniv Ovadia Emily Fertig Jie Ren Zachary Nado D. Sculley Sebastian Nowozin Joshua Dillon Balaji Lakshminarayanan and Jasper Snoek. 2019. Can you trust your models uncertainty? Evaluating predictive uncertainty under dataset shift. Advances in Neural Information Processing Systems 13991\u201314002."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.5555\/1953048.2078195"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_1_32_1","volume-title":"Reliable Machine Learning in the Wild Workshop, 34th International Conference on Machine Learning. arxiv:1707","author":"Rauber Jonas","year":"2017","unstructured":"Jonas Rauber , Wieland Brendel , and Matthias Bethge . 2017 . Foolbox: A Python toolbox to benchmark the robustness of machine learning models . In Reliable Machine Learning in the Wild Workshop, 34th International Conference on Machine Learning. arxiv:1707 .04131 Jonas Rauber, Wieland Brendel, and Matthias Bethge. 2017. Foolbox: A Python toolbox to benchmark the robustness of machine learning models. In Reliable Machine Learning in the Wild Workshop, 34th International Conference on Machine Learning. arxiv:1707.04131"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.21105\/joss.02607"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-020-09881-0"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/32.962562"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3221269.3223036"},{"key":"e_1_3_2_1_37_1","volume-title":"ISSTA 2022 - Technical Track Call for Papers - Section on Replicability Studies. https:\/\/conf.researchr.org\/track\/issta-2022\/issta-2022-technical-papers#Call-for-Papers Accessed: 2021-15-12","author":"Smaragdakis Yannis","unstructured":"Yannis Smaragdakis . [n. d.]. ISSTA 2022 - Technical Track Call for Papers - Section on Replicability Studies. https:\/\/conf.researchr.org\/track\/issta-2022\/issta-2022-technical-papers#Call-for-Papers Accessed: 2021-15-12 Yannis Smaragdakis. [n. d.]. ISSTA 2022 - Technical Track Call for Papers - Section on Replicability Studies. https:\/\/conf.researchr.org\/track\/issta-2022\/issta-2022-technical-papers#Call-for-Papers Accessed: 2021-15-12"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSREW51248.2020.00073"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380353"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSMCB.2012.2236828"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/DeepTest52559.2021.00009"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST49551.2021.00015"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST49551.2021.00056"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.6520942"},{"key":"e_1_3_2_1_45_1","volume-title":"Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning Algorithms. CoRR, abs\/1708.07747","author":"Xiao Han","year":"2017","unstructured":"Han Xiao , Kashif Rasul , and Roland Vollgraf . 2017. Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning Algorithms. CoRR, abs\/1708.07747 ( 2017 ), arxiv:1708.07747. arxiv:1708.07747 Han Xiao, Kashif Rasul, and Roland Vollgraf. 2017. Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning Algorithms. CoRR, abs\/1708.07747 (2017), arxiv:1708.07747. arxiv:1708.07747"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380368"}],"event":{"name":"ISSTA '22: 31st ACM SIGSOFT International Symposium on Software Testing and Analysis","location":"Virtual South Korea","acronym":"ISSTA '22","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 31st ACM SIGSOFT International Symposium on Software Testing and Analysis"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3533767.3534375","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3533767.3534375","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T18:43:40Z","timestamp":1750272220000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3533767.3534375"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,7,18]]},"references-count":46,"alternative-id":["10.1145\/3533767.3534375","10.1145\/3533767"],"URL":"https:\/\/doi.org\/10.1145\/3533767.3534375","relation":{},"subject":[],"published":{"date-parts":[[2022,7,18]]},"assertion":[{"value":"2022-07-18","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}