{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T05:03:30Z","timestamp":1750309410826,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":33,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T00:00:00Z","timestamp":1726012800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,9,11]]},"DOI":"10.1145\/3650212.3680401","type":"proceedings-article","created":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T11:44:25Z","timestamp":1726055065000},"page":"1300-1311","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Evaluating Deep Neural Networks in Deployment: A Comparative Study (Replicability Study)"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0729-4684","authenticated-orcid":false,"given":"Eduard","family":"Pinconschi","sequence":"first","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1242-7701","authenticated-orcid":false,"given":"Divya","family":"Gopinath","sequence":"additional","affiliation":[{"name":"KBR Inc., Mountain View, USA \/ NASA Ames, Mountain View, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3734-3157","authenticated-orcid":false,"given":"Rui","family":"Abreu","sequence":"additional","affiliation":[{"name":"INESC-ID, Porto, Portugal \/ University of Porto, Porto, Portugal"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5579-6961","authenticated-orcid":false,"given":"Corina S.","family":"P\u0103s\u0103reanu","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Mountain View, USA \/ NASA Ames, Mountain View, USA"}]}],"member":"320","published-online":{"date-parts":[[2024,9,11]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"[n. d.]. Keras documentation: CIFAR10 small images classification dataset. https:\/\/keras.io\/api\/datasets\/cifar10\/ Accessed on 2024-09-11"},{"key":"e_1_3_2_1_2_1","unstructured":"2016. Pima Indians Diabetes Dataset. https:\/\/www.kaggle.com\/datasets\/uciml\/pima-indians-diabetes-database Accessed on 2024-09-11"},{"key":"e_1_3_2_1_3_1","unstructured":"2018. Bank Marketing Dataset. https:\/\/www.kaggle.com\/datasets\/sharanmk\/bank-marketing-term-deposit Accessed on 2024-09-11"},{"key":"e_1_3_2_1_4_1","unstructured":"2019. German Credit Risk Classification with Keras. https:\/\/www.kaggle.com\/code\/twunderbar\/german-credit-risk-classification-with-keras\/data Accessed on 2024-09-11"},{"key":"e_1_3_2_1_5_1","unstructured":"2019. House Price Prediction Dataset. https:\/\/www.kaggle.com\/datasets\/moewie94\/housepricedata Accessed on 2024-09-11"},{"key":"e_1_3_2_1_6_1","unstructured":"2019. Kaggle Notebook for preprocessing German Credit dataset. https:\/\/www.kaggle.com\/code\/twunderbar\/german-credit-risk-classification-with-keras\/notebook Accessed on 2024-09-11"},{"key":"e_1_3_2_1_7_1","unstructured":"2019. Kaggle Notebook for preprocessing House Price dataset. https:\/\/www.kaggle.com\/code\/moewie94\/neural-network-with-keras\/notebook Accessed on 2024-09-11"},{"key":"e_1_3_2_1_8_1","unstructured":"2021. Kaggle Notebook for preprocessing Bank Marketing dataset. https:\/\/www.kaggle.com\/code\/rxsraghavagrawal\/build-your-first-ann-using-keras-on-bank-customer Accessed on 2024-09-11"},{"key":"e_1_3_2_1_9_1","unstructured":"Shibbir Ahmed Hongyang Gao and Hridesh Rajan. 2023. Repository of DeepInfer.. https:\/\/github.com\/shibbirtanvin\/DeepInfer"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"crossref","unstructured":"Shibbir Ahmed Hongyang Gao and Hridesh Rajan. 2024. Inferring Data Preconditions from Deep Learning Models for Trustworthy Prediction in Deployment. arxiv:2401.14628.","DOI":"10.1145\/3597503.3623333"},{"key":"e_1_3_2_1_11_1","volume-title":"The advantages of the Matthews correlation coefficient (MCC) over F1 score and accuracy in binary classification evaluation. BMC Genomics, 21","author":"Chicco Davide","year":"2020","unstructured":"Davide Chicco and Giuseppe Jurman. 2020. The advantages of the Matthews correlation coefficient (MCC) over F1 score and accuracy in binary classification evaluation. BMC Genomics, 21 (2020), https:\/\/api.semanticscholar.org\/CorpusID:209528322"},{"key":"e_1_3_2_1_12_1","volume-title":"Proceedings of the 25th International Conference on Neural Information Processing Systems -","volume":"2","author":"Cire\u015fan Dan C.","year":"2012","unstructured":"Dan C. Cire\u015fan, Alessandro Giusti, Luca M. Gambardella, and J\u00fcrgen Schmidhuber. 2012. Deep neural networks segment neuronal membranes in electron microscopy images. In Proceedings of the 25th International Conference on Neural Information Processing Systems - Volume 2 (NIPS\u201912). Curran Associates Inc., Red Hook, NY, USA. 2843\u20132851."},{"key":"e_1_3_2_1_13_1","volume-title":"Advances in Neural Information Processing Systems, H. Wallach, H. Larochelle, A. Beygelzimer, F. d' Alch\u00e9-Buc","author":"Corbi\u00e8re Charles","year":"2019","unstructured":"Charles Corbi\u00e8re, Nicolas THOME, Avner Bar-Hen, Matthieu Cord, and Patrick P\u00e9rez. 2019. Addressing Failure Prediction by Learning Model Confidence. In Advances in Neural Information Processing Systems, H. Wallach, H. Larochelle, A. Beygelzimer, F. d' Alch\u00e9-Buc, E. Fox, and R. Garnett (Eds.). 32, Curran Associates, Inc.. https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2019\/file\/757f843a169cc678064d9530d12a1881-Paper.pdf"},{"key":"e_1_3_2_1_14_1","volume-title":"Data Invariants: On Trust in Data-Driven Systems. CoRR, abs\/2003.01289","author":"Fariha Anna","year":"2020","unstructured":"Anna Fariha, Ashish Tiwari, Arjun Radhakrishna, Sumit Gulwani, and Alexandra Meliou. 2020. Data Invariants: On Trust in Data-Driven Systems. CoRR, abs\/2003.01289 (2020), arXiv:2003.01289. arxiv:2003.01289"},{"key":"e_1_3_2_1_15_1","volume-title":"Property Inference for Deep Neural Networks. 2019 34th IEEE\/ACM International Conference on Automated Software Engineering (ASE), 797\u2013809","author":"Gopinath Divya","year":"2019","unstructured":"Divya Gopinath, Hayes Converse, Corina S. P\u0103s\u0103reanu, and Ankur Taly. 2019. Property Inference for Deep Neural Networks. 2019 34th IEEE\/ACM International Conference on Automated Software Engineering (ASE), 797\u2013809. https:\/\/api.semanticscholar.org\/CorpusID:202577825"},{"volume-title":"The Benchmark Handbook: For Database and Transaction Processing Systems","author":"Gray J.","key":"e_1_3_2_1_16_1","unstructured":"J. Gray. 1991. The Benchmark Handbook: For Database and Transaction Processing Systems. M. Kaufmann Publishers. isbn:9781558601598 lccn:91012259 https:\/\/books.google.com\/books?id=UKQXAAAAIAAJ"},{"key":"e_1_3_2_1_17_1","volume-title":"A Baseline for Detecting Misclassified and Out-of-Distribution Examples in Neural Networks. CoRR, abs\/1610.02136","author":"Hendrycks Dan","year":"2016","unstructured":"Dan Hendrycks and Kevin Gimpel. 2016. A Baseline for Detecting Misclassified and Out-of-Distribution Examples in Neural Networks. CoRR, abs\/1610.02136 (2016), arXiv:1610.02136. arxiv:1610.02136"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1561\/0600000079"},{"key":"e_1_3_2_1_19_1","unstructured":"Heinrich Jiang Been Kim Melody Y. Guan and Maya Gupta. 2018. To Trust Or Not To Trust A Classifier. arxiv:1805.11783."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1002\/ISAF.1500"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-95561-8_7"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01114"},{"key":"e_1_3_2_1_23_1","unstructured":"Alex Krizhevsky. 2009. Learning Multiple Layers of Features from Tiny Images. https:\/\/api.semanticscholar.org\/CorpusID:18268744"},{"key":"e_1_3_2_1_24_1","volume-title":"Pasareanu","author":"Pinconschi Eduard","year":"2024","unstructured":"Eduard Pinconschi, Divya Gopinath, Rui Abreu, and Corina S. Pasareanu. 2024. Evaluating Deep Neural Networks in Deployment (A Comparative and Replicability Study). arxiv:2407.08730. arxiv:2407.08730"},{"key":"e_1_3_2_1_25_1","volume-title":"Johnson","author":"Roberts Don","year":"2004","unstructured":"Don Roberts and Ralph E. Johnson. 2004. Evolving Frameworks A Pattern Language for Developing Object-Oriented Frameworks. https:\/\/api.semanticscholar.org\/CorpusID:10900135"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380353"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-81685-8_1"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-17196-3_4"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3617574.3622747"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2019.8683359"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380379"},{"key":"e_1_3_2_1_32_1","volume-title":"Self-Checking Deep Neural Networks in Deployment. 2021 IEEE\/ACM 43rd International Conference on Software Engineering (ICSE), 372\u2013384","author":"Xiao Yan","year":"2021","unstructured":"Yan Xiao, Ivan Beschastnikh, David S. Rosenblum, Changsheng Sun, Sebastian G. Elbaum, Yun Lin, and Jin Song Dong. 2021. Self-Checking Deep Neural Networks in Deployment. 2021 IEEE\/ACM 43rd International Conference on Software Engineering (ICSE), 372\u2013384. https:\/\/api.semanticscholar.org\/CorpusID:230125000"},{"key":"e_1_3_2_1_33_1","volume-title":"Yun Lin and Jin Song Dong","author":"Xiao Yan","year":"2021","unstructured":"Yan Xiao, Ivan Beschastnikhy, David S. Rosenblumz, Changsheng Sun, Sebastian Elbaum, Yun Lin and Jin Song Dong. 2021. Repository of SelfChecker.. https:\/\/github.com\/self-checker\/SelfChecker"}],"event":{"name":"ISSTA '24: 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","AITO"],"location":"Vienna Austria","acronym":"ISSTA '24"},"container-title":["Proceedings of the 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650212.3680401","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3650212.3680401","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:58:09Z","timestamp":1750294689000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650212.3680401"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,11]]},"references-count":33,"alternative-id":["10.1145\/3650212.3680401","10.1145\/3650212"],"URL":"https:\/\/doi.org\/10.1145\/3650212.3680401","relation":{},"subject":[],"published":{"date-parts":[[2024,9,11]]},"assertion":[{"value":"2024-09-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}