{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,9]],"date-time":"2026-07-09T05:22:42Z","timestamp":1783574562742,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":58,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,10,27]],"date-time":"2024-10-27T00:00:00Z","timestamp":1729987200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62372227"],"award-info":[{"award-number":["62372227"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,10,27]]},"DOI":"10.1145\/3691620.3695498","type":"proceedings-article","created":{"date-parts":[[2024,10,18]],"date-time":"2024-10-18T15:39:19Z","timestamp":1729265959000},"page":"1219-1231","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Prioritizing Test Inputs for DNNs Using Training Dynamics"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-6326-5352","authenticated-orcid":false,"given":"Jian","family":"Shen","sequence":"first","affiliation":[{"name":"Nanjing University, Nanjing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3849-3416","authenticated-orcid":false,"given":"Zhong","family":"Li","sequence":"additional","affiliation":[{"name":"Nanjing University, Nanjing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4011-5350","authenticated-orcid":false,"given":"Minxue","family":"Pan","sequence":"additional","affiliation":[{"name":"Nanjing University, Nanjing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3090-9568","authenticated-orcid":false,"given":"Xuandong","family":"Li","sequence":"additional","affiliation":[{"name":"Nanjing University, Nanjing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,10,27]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Critical Learning Periods in Deep Networks. In 7th International Conference on Learning Representations, ICLR 2019","author":"Achille Alessandro","year":"2019","unstructured":"Alessandro Achille, Matteo Rovere, and Stefano Soatto. 2019. Critical Learning Periods in Deep Networks. In 7th International Conference on Learning Representations, ICLR 2019, New Orleans, LA, USA, May 6--9, 2019. OpenReview.net. https:\/\/openreview.net\/forum?id=BkeStsCcKQ"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/2034691.2034742"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1016\/J.KNOSYS.2019.105210"},{"key":"e_1_3_2_1_4_1","first-page":"10876","article-title":"Deep learning through the lens of example difficulty","volume":"34","author":"Baldock Robert","year":"2021","unstructured":"Robert Baldock, Hartmut Maennel, and Behnam Neyshabur. 2021. Deep learning through the lens of example difficulty. Advances in Neural Information Processing Systems 34 (2021), 10876--10889.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/AITEST.2019.000-6"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380389"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3394112"},{"key":"e_1_3_2_1_8_1","volume-title":"Deep Neural Network Test Coverage: How Far Are We? CoRR abs\/2010.04946","author":"Chen Junjie","year":"2020","unstructured":"Junjie Chen, Ming Yan, Zan Wang, Yuning Kang, and Zhuo Wu. 2020. Deep Neural Network Test Coverage: How Far Are We? CoRR abs\/2010.04946 (2020). arXiv:2010.04946 https:\/\/arxiv.org\/abs\/2010.04946"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"crossref","unstructured":"Tianqi Chen and Carlos Guestrin. 2016. XGBoost: A Scalable Tree Boosting System. In KDD. ACM 785--794.","DOI":"10.1145\/2939672.2939785"},{"key":"e_1_3_2_1_10_1","unstructured":"Norman Cliff. 1996. Ordinal methods for behavioral data analysis."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.asoc.2012.07.029"},{"key":"e_1_3_2_1_12_1","volume-title":"On the Learning Dynamics of Deep Neural Networks. CoRR abs\/1809.06848","author":"des Combes Remi Tachet","year":"2018","unstructured":"Remi Tachet des Combes, Mohammad Pezeshki, Samira Shabanian, Aaron C. Courville, and Yoshua Bengio. 2018. On the Learning Dynamics of Deep Neural Networks. CoRR abs\/1809.06848 (2018). arXiv:1809.06848 http:\/\/arxiv.org\/abs\/1809.06848"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-16722-6_10"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3395363.3397357"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"Artem Grotov and Maarten de Rijke. 2016. Online Learning to Rank for Information Retrieval: SIGIR 2016 Tutorial. In SIGIR. ACM 1215--1218.","DOI":"10.1145\/2911451.2914798"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/SANER56733.2023.00056"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_1_19_1","volume-title":"7th International Conference on Learning Representations, ICLR 2019","author":"Hendrycks Dan","year":"2019","unstructured":"Dan Hendrycks and Thomas G. Dietterich. 2019. Benchmarking Neural Network Robustness to Common Corruptions and Perturbations. In 7th International Conference on Learning Representations, ICLR 2019, New Orleans, LA, USA, May 6--9, 2019. OpenReview.net. https:\/\/openreview.net\/forum?id=HJz6tiCqYm"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511598"},{"key":"e_1_3_2_1_21_1","volume-title":"Aries: Efficient Testing of Deep Neural Networks via Labeling-Free Accuracy Estimation","author":"Hu Qiang","year":"2023","unstructured":"Qiang Hu, Yuejun Guo, Xiaofei Xie, Maxime Cordy, Mike Papadakis, Lei Ma, and Yves Le Traon. 2023. Aries: Efficient Testing of Deep Neural Networks via Labeling-Free Accuracy Estimation. In ICSE. IEEE, 1776--1787."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2019.00126"},{"key":"e_1_3_2_1_23_1","volume-title":"Weinberger","author":"Huang Gao","year":"2016","unstructured":"Gao Huang, Zhuang Liu, and Kilian Q. Weinberger. 2016. Densely Connected Convolutional Networks. CoRR abs\/1608.06993 (2016)."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/1328964.1328974"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_2_1_26_1","unstructured":"Alex Krizhevsky Geoffrey Hinton et al. 2009. Learning multiple layers of features from tiny images. (2009)."},{"key":"e_1_3_2_1_27_1","volume-title":"Hinton","author":"Krizhevsky Alex","year":"2012","unstructured":"Alex Krizhevsky, Ilya Sutskever, and Geoffrey E. Hinton. 2012. ImageNet Classification with Deep Convolutional Neural Networks. In Advances in Neural Information Processing Systems 25: 26th Annual Conference on Neural Information Processing Systems 2012. Proceedings of a meeting held December 3--6, 2012, Lake Tahoe, Nevada, United States, Peter L. Bartlett, Fernando C. N. Pereira, Christopher J. C. Burges, L\u00e9on Bottou, and Kilian Q. Weinberger (Eds.). 1106--1114. https:\/\/proceedings.neurips.cc\/paper\/2012\/hash\/c399862d3b9d6b76c8436e924a68c45b-Abstract.html"},{"key":"e_1_3_2_1_28_1","volume-title":"Tiny imagenet visual recognition challenge. CS 231N 7, 7","author":"Le Ya","year":"2015","unstructured":"Ya Le and Xuan Yang. 2015. Tiny imagenet visual recognition challenge. CS 231N 7, 7 (2015), 3."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1038\/NATURE14539"},{"key":"e_1_3_2_1_30_1","volume-title":"Advances in Neural Information Processing Systems 34: Annual Conference on Neural Information Processing Systems 2021","author":"Li Yu","year":"2021","unstructured":"Yu Li, Min Li, Qiuxia Lai, Yannan Liu, and Qiang Xu. 2021. TestRank: Bringing Order into Unlabeled Test Instances for Deep Learning Tasks. In Advances in Neural Information Processing Systems 34: Annual Conference on Neural Information Processing Systems 2021, NeurIPS 2021, December 6--14, 2021, virtual, Marc'Aurelio Ranzato, Alina Beygelzimer, Yann N. Dauphin, Percy Liang, and Jennifer Wortman Vaughan (Eds.). 20874--20886. https:\/\/proceedings.neurips.cc\/paper\/2021\/hash\/ae78510109d46b0a6eef9820a4ca95d6-Abstract.html"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2015.7381798"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3417330"},{"key":"e_1_3_2_1_34_1","volume-title":"Learning Word Vectors for Sentiment Analysis","author":"Maas Andrew L.","unstructured":"Andrew L. Maas, Raymond E. Daly, Peter T. Pham, Dan Huang, Andrew Y. Ng, and Christopher Potts. 2011. Learning Word Vectors for Sentiment Analysis. In ACL. The Association for Computer Linguistics, 142--150."},{"key":"e_1_3_2_1_35_1","volume-title":"ICML 2019 Workshop on Identifying and Understanding Deep Learning Phenomena.","author":"Mangalam Karttikeya","year":"2019","unstructured":"Karttikeya Mangalam and Vinay Uday Prabhu. 2019. Do deep neural networks learn shallow learnable examples first?. In ICML 2019 Workshop on Identifying and Understanding Deep Learning Phenomena."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-24769-9_32"},{"key":"e_1_3_2_1_37_1","volume-title":"PyTorch: An Imperative Style","author":"Paszke Adam","unstructured":"Adam Paszke, Sam Gross, Francisco Massa, Adam Lerer, James Bradbury, Gregory Chanan, Trevor Killeen, Zeming Lin, Natalia Gimelshein, Luca Antiga, Alban Desmaison, Andreas K\u00f6pf, Edward Z. Yang, Zachary DeVito, Martin Raison, Alykhan Tejani, Sasank Chilamkurthy, Benoit Steiner, Lu Fang, Junjie Bai, and Soumith Chintala. 2019. PyTorch: An Imperative Style, High-Performance Deep Learning Library. In NeurIPS. 8024--8035."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_1_39_1","volume-title":"Gradient Starvation: A Learning Proclivity in Neural Networks. In Advances in Neural Information Processing Systems 34: Annual Conference on Neural Information Processing Systems 2021","author":"Pezeshki Mohammad","year":"2021","unstructured":"Mohammad Pezeshki, S\u00e9kou-Oumar Kaba, Yoshua Bengio, Aaron C. Courville, Doina Precup, and Guillaume Lajoie. 2021. Gradient Starvation: A Learning Proclivity in Neural Networks. In Advances in Neural Information Processing Systems 34: Annual Conference on Neural Information Processing Systems 2021, NeurIPS 2021, December 6--14, 2021, virtual, Marc'Aurelio Ranzato, Alina Beygelzimer, Yann N. Dauphin, Percy Liang, and Jennifer Wortman Vaughan (Eds.). 1256--1272. https:\/\/proceedings.neurips.cc\/paper\/2021\/hash\/0987b8b338d6c90bbedd8631bc499221-Abstract.html"},{"key":"e_1_3_2_1_40_1","volume-title":"Proceedings of the 35th Annual Meeting of the Cognitive Science Society, CogSci 2013","author":"Saxe Andrew M.","year":"2013","unstructured":"Andrew M. Saxe, James L. McClelland, and Surya Ganguli. 2013. Learning hierarchical categories in deep neural networks. In Proceedings of the 35th Annual Meeting of the Cognitive Science Society, CogSci 2013, Berlin, Germany, July 31 - August 3, 2013, Markus Knauff, Michael Pauen, Natalie Sebanz, and Ipke Wachsmuth (Eds.). cognitivesciencesociety.org. https:\/\/escholarship.org\/uc\/item\/2fv5q3hn"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/78.650093"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416621"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1002\/STVR.1695"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1016\/J.INS.2020.03.022"},{"key":"e_1_3_2_1_45_1","volume-title":"3rd International Conference on Learning Representations, ICLR","author":"Simonyan Karen","year":"2015","unstructured":"Karen Simonyan and Andrew Zisserman. 2015. Very Deep Convolutional Networks for Large-Scale Image Recognition. In 3rd International Conference on Learning Representations, ICLR 2015, San Diego, CA, USA, May 7--9, 2015, Conference Track Proceedings, Yoshua Bengio and Yann LeCun (Eds.). http:\/\/arxiv.org\/abs\/1409.1556"},{"key":"e_1_3_2_1_46_1","first-page":"30","article-title":"Tesla's autopilot was involved in another deadly car crash","volume":"3","author":"Stewart Jack","year":"2018","unstructured":"Jack Stewart. 2018. Tesla's autopilot was involved in another deadly car crash. Wired 3 (2018), 30.","journal-title":"Wired"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238172"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00046"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3533767.3534375"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-89960-2_22"},{"key":"e_1_3_2_1_52_1","volume-title":"Individual Comparisons by Ranking Methods. Biometrics","author":"Wilcoxon Frank","year":"1945","unstructured":"Frank Wilcoxon. 1945. Individual Comparisons by Ranking Methods. Biometrics (1945)."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330579"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2962027"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1109\/TASE49443.2020.00020"},{"key":"e_1_3_2_1_56_1","volume-title":"A Noise-Sensitivity-Analysis-Based Test Prioritization Technique for Deep Neural Networks. CoRR abs\/1901.00054","author":"Zhang Long","year":"2019","unstructured":"Long Zhang, Xuechao Sun, Yong Li, and Zhenyu Zhang. 2019. A Noise-Sensitivity-Analysis-Based Test Prioritization Technique for Deep Neural Networks. CoRR abs\/1901.00054 (2019). arXiv:1901.00054 http:\/\/arxiv.org\/abs\/1901.00054"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE56229.2023.00126"},{"key":"e_1_3_2_1_58_1","volume-title":"A google self-driving car caused a crash for the first time. The Verge 198","author":"Ziegler Chris","year":"2016","unstructured":"Chris Ziegler. 2016. A google self-driving car caused a crash for the first time. The Verge 198 (2016)."}],"event":{"name":"ASE '24: 39th IEEE\/ACM International Conference on Automated Software Engineering","location":"Sacramento CA USA","acronym":"ASE '24","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 39th IEEE\/ACM International Conference on Automated Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3691620.3695498","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3691620.3695498","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:06:19Z","timestamp":1750291579000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3691620.3695498"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,27]]},"references-count":58,"alternative-id":["10.1145\/3691620.3695498","10.1145\/3691620"],"URL":"https:\/\/doi.org\/10.1145\/3691620.3695498","relation":{},"subject":[],"published":{"date-parts":[[2024,10,27]]},"assertion":[{"value":"2024-10-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}