{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,27]],"date-time":"2025-08-27T15:49:19Z","timestamp":1756309759082,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":61,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,3,31]],"date-time":"2025-03-31T00:00:00Z","timestamp":1743379200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100000038","name":"Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,3,31]]},"DOI":"10.1145\/3672608.3707978","type":"proceedings-article","created":{"date-parts":[[2025,5,14]],"date-time":"2025-05-14T18:26:21Z","timestamp":1747247181000},"page":"1548-1559","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["D-semble: Efficient Diversity-Guided Search for Resilient ML Ensembles"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7260-4124","authenticated-orcid":false,"given":"Abraham","family":"Chan","sequence":"first","affiliation":[{"name":"The University of British Columbia, Vancouver, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2949-6826","authenticated-orcid":false,"given":"Arpan","family":"Gujarati","sequence":"additional","affiliation":[{"name":"The University of British Columbia, Vancouver, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2380-3415","authenticated-orcid":false,"given":"Karthik","family":"Pattabiraman","sequence":"additional","affiliation":[{"name":"The University of British Columbia, Vancouver, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2959-4802","authenticated-orcid":false,"given":"Sathish","family":"Gopalakrishnan","sequence":"additional","affiliation":[{"name":"The University of British Columbia, Vancouver, Canada"}]}],"member":"320","published-online":{"date-parts":[[2025,5,14]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proc. of ICCIDS.","author":"Nabizath Saleena Ankit","year":"2018","unstructured":"Ankit and Nabizath Saleena. 2018. An Ensemble Classification System for Twitter Sentiment Analysis. In Proc. of ICCIDS."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/DSN.2018.00066"},{"key":"e_1_3_2_1_4_1","volume-title":"Stacked regressions. Mach Learn","author":"Breiman L.","year":"1996","unstructured":"L. Breiman. 1996. Stacked regressions. Mach Learn (1996)."},{"key":"e_1_3_2_1_5_1","volume-title":"of Radiologists","author":"Assoc Canadian","year":"2019","unstructured":"Canadian Assoc. of Radiologists. 2019. CAR White Paper on Ethical and Legal Issues Related to Artificial Intelligence in Radiology. CAR Journal (2019)."},{"key":"e_1_3_2_1_6_1","volume":"202","author":"Casado-Garc\u00eda A.","unstructured":"A. Casado-Garc\u00eda and J. Heras. 2020. Ensemble Methods for Object Detection. In Proc. of ECAI.","journal-title":"J. Heras."},{"volume-title":"Proc. of QRS.","author":"Abraham","key":"e_1_3_2_1_7_1","unstructured":"Abraham Chan et al. 2021. Understanding the Resilience of Neural Network Ensembles against Faulty Training Data. In Proc. of QRS."},{"volume-title":"Proc. of DSN.","author":"Abraham","key":"e_1_3_2_1_8_1","unstructured":"Abraham Chan et al. 2022. The Fault in Our Data Stars: Studying Mitigation Techniques against Faulty Training Data in Machine Learning Applications. In Proc. of DSN."},{"volume-title":"Proc. of CHI.","author":"Chee Joseph","key":"e_1_3_2_1_9_1","unstructured":"Joseph Chee Chang et al. 2017. Revolt: Collaborative Crowdsourcing for Labeling Machine Learning Datasets. In Proc. of CHI."},{"key":"e_1_3_2_1_10_1","volume-title":"Proc. of FTCS'78","author":"Chen Liming","year":"1978","unstructured":"Liming Chen and Algirdas Avizienis. 1978. N-version programming: A fault-tolerance approach to reliability of software operation. In Proc. of FTCS'78."},{"volume-title":"Proc. of MUC.","author":"Nancy","key":"e_1_3_2_1_11_1","unstructured":"Nancy Chinchor et al. 1993. MUC-5 evaluation metrics. In Proc. of MUC."},{"key":"e_1_3_2_1_12_1","unstructured":"Cleanlab. 2024. Cleanlab. https:\/\/github.com\/cleanlab\/cleanlab."},{"volume-title":"Proc. of CVPR.","author":"Jia","key":"e_1_3_2_1_13_1","unstructured":"Jia Deng et al. 2009. ImageNet: A Large-Scale Hierarchical Image Database. In Proc. of CVPR."},{"key":"e_1_3_2_1_14_1","unstructured":"Brad Dwyer. 2020. A popular self-driving car dataset is missing labels for hundreds of pedestrians."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1049\/ip-sen:19982444"},{"key":"e_1_3_2_1_16_1","volume-title":"Exploring Very Large State Spaces Using Genetic Algorithms. STTT","author":"Godefroid Patrice","year":"2002","unstructured":"Patrice Godefroid and Sarfraz Khurshid. 2002. Exploring Very Large State Spaces Using Genetic Algorithms. STTT (2002)."},{"key":"e_1_3_2_1_17_1","unstructured":"Muhammad Hamzah. 2020. https:\/\/github.com\/mdhmz1\/Auto-Annotate."},{"volume-title":"Proc. of ECOOP.","author":"Dan","key":"e_1_3_2_1_18_1","unstructured":"Dan Hao et al. 2013. Is This a Bug or an Obsolete Test?. In Proc. of ECOOP."},{"key":"e_1_3_2_1_19_1","volume-title":"Genetic Algorithms. Scientific American","author":"Holland John H.","year":"1992","unstructured":"John H. Holland. 1992. Genetic Algorithms. Scientific American (1992)."},{"key":"e_1_3_2_1_20_1","volume-title":"Crowdsourcing: A Study of Annotation Selection Criteria. In Proc. of HLT.","author":"Hsueh Pei-Yun","year":"2009","unstructured":"Pei-Yun Hsueh et al. 2009. Data Quality from Crowdsourcing: A Study of Annotation Selection Criteria. In Proc. of HLT."},{"volume-title":"Proc. of ICLR.","author":"Gao","key":"e_1_3_2_1_21_1","unstructured":"Gao Huang et al. 2017. Snapshot Ensembles: Train 1, get M for free. In Proc. of ICLR."},{"volume-title":"Proc. of SIGMOD.","author":"Daniel","key":"e_1_3_2_1_22_1","unstructured":"Daniel Kang et al. 2022. Finding Label and Model Errors in Perception Data With Learned Observation Assertions. In Proc. of SIGMOD."},{"key":"e_1_3_2_1_23_1","unstructured":"Daniel Kermany et al. 2018. Labeled Optical Coherence Tomography (OCT) and Chest X-Ray Images for Classification. https:\/\/data.mendeley.com\/datasets\/rscbjbr9sj\/2."},{"key":"e_1_3_2_1_24_1","unstructured":"R. Kesten et al. 2020. Lyft L5 Perception Dataset. https:\/\/level5.lyft.com\/dataset\/"},{"key":"e_1_3_2_1_25_1","volume-title":"Jason Van Hulse, and Amri Napolitano","author":"Khoshgoftaar Taghi","year":"2011","unstructured":"Taghi Khoshgoftaar, Jason Van Hulse, and Amri Napolitano. 2011. Comparing Boosting and Bagging Techniques With Noisy and Imbalanced Data. Trans. on Sys, Man, and Cyb. (2011)."},{"key":"e_1_3_2_1_26_1","volume-title":"Overview, Definition, and Architecture","author":"Kreuzberger Dominik","year":"2022","unstructured":"Dominik Kreuzberger, Niklas K\u00fchl, and Sebastian Hirschl. 2022. MLOps: Overview, Definition, and Architecture. IEEE Access (2022)."},{"key":"e_1_3_2_1_28_1","unstructured":"Ludmila Kuncheva et al. 2003. Measures of Diversity in Classifier Ensembles and Their Relationship with the Ensemble Accuracy. Mach Learn (2003)."},{"volume-title":"Proc. of S&P.","author":"Mathias","key":"e_1_3_2_1_29_1","unstructured":"Mathias Lecuyer et al. 2019. Certified Robustness to Adversarial Examples with Differential Privacy. In Proc. of S&P."},{"volume-title":"Proc. of CVPR.","author":"Kuang-Huei","key":"e_1_3_2_1_30_1","unstructured":"Kuang-Huei Lee et al. 2018. CleanNet: Transfer Learning for Scalable Image Classifier Training with Label Noise. In Proc. of CVPR."},{"key":"e_1_3_2_1_31_1","unstructured":"Tsung-Yi Lin et al. 2014. Microsoft COCO: Common Objects in Context. arXiv:1405.0312."},{"key":"e_1_3_2_1_32_1","volume-title":"Proc. of ICLR.","author":"Loshchilov Ilya","year":"2017","unstructured":"Ilya Loshchilov and Frank Hutter. 2017. SGDR: Stochastic Gradient Descent with Warm Restarts. In Proc. of ICLR."},{"volume-title":"Proc. of GECCO.","author":"Zhichao","key":"e_1_3_2_1_33_1","unstructured":"Zhichao Lu et al. 2019. NSGA-Net: Neural Architecture Search Using Multi-Objective Genetic Algorithm. In Proc. of GECCO."},{"key":"e_1_3_2_1_34_1","volume-title":"Aditya Krishna Menon, and Sanjiv Kumar","author":"Lukasik Michal","year":"2020","unstructured":"Michal Lukasik, Srinadh Bhojanapalli, Aditya Krishna Menon, and Sanjiv Kumar. 2020. Does label smoothing mitigate label noise? arXiv:2003.02819."},{"volume-title":"Proc. of ICML.","author":"Xingjun","key":"e_1_3_2_1_35_1","unstructured":"Xingjun Ma et al. 2020. Normalized Loss Functions for Deep Learning with Noisy Labels. In Proc. of ICML."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/DSN-W.2019.00017"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"crossref","unstructured":"Zbigniew Michalewicz et al. 1996. Evolutionary Algorithms for Constrained Parameter Optimization Problems. Evolutionary Computation (1996).","DOI":"10.1162\/evco.1996.4.1.1"},{"key":"e_1_3_2_1_38_1","unstructured":"Jamie Murdoch. 2022. https:\/\/medium.com\/p\/79d382edf22b."},{"volume-title":"Proc. of WAIN.","author":"Sasu","key":"e_1_3_2_1_39_1","unstructured":"Sasu M\u00e4kinen et al. 2021. Who Needs MLOps: What Data Scientists Seek to Accomplish and How Can MLOps Help?. In Proc. of WAIN."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/DeepTest52559.2021.00010"},{"key":"e_1_3_2_1_41_1","volume-title":"Northcutt et al","author":"Curtis","year":"2021","unstructured":"Curtis G. Northcutt et al. 2021. Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks. arXiv:2103.14749."},{"key":"e_1_3_2_1_42_1","volume-title":"Northcutt et al","author":"Curtis","year":"2022","unstructured":"Curtis G. Northcutt et al. 2022. Confident Learning: Estimating Uncertainty in Dataset Labels. In Proc. of NeurIPS."},{"key":"e_1_3_2_1_43_1","volume-title":"On the Interpretation and Disaggregation of Gini Coefficients. The Economic Journal","author":"Pyatt Graham","year":"1976","unstructured":"Graham Pyatt. 1976. On the Interpretation and Disaggregation of Gini Coefficients. The Economic Journal (1976)."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/2568225.2568254"},{"key":"e_1_3_2_1_45_1","volume-title":"Your Machine Learning Tools Are Useless. Harvard Business Review","author":"Redman Thomas C","year":"2018","unstructured":"Thomas C Redman. 2018. If Your Data is Bad, Your Machine Learning Tools Are Useless. Harvard Business Review (2018)."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"crossref","unstructured":"Jonathan G Richens et al. 2020. Improving the accuracy of medical diagnosis with causal machine learning. Nature Communications (2020).","DOI":"10.1038\/s41467-020-17419-7"},{"volume-title":"Proc. of CHI.","author":"Nithya","key":"e_1_3_2_1_47_1","unstructured":"Nithya Sambasivan et al. 2021. \"Everyone wants to do the model work, not the data work\": Data Cascades in High-Stakes AI. In Proc. of CHI."},{"volume-title":"Proc. of WACV.","author":"Christian","key":"e_1_3_2_1_48_1","unstructured":"Christian Simon et al. 2022. Towards a Robust Differentiable Architecture Search under Label Noise. In Proc. of WACV."},{"key":"e_1_3_2_1_49_1","volume-title":"Proc. of ICML.","author":"Song Hwanjun","year":"2019","unstructured":"Hwanjun Song, Minseok Kim, and Jae-Gil Lee. 2019. SELFIE: Refurbishing Unclean Samples for Robust Deep Learning. In Proc. of ICML."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"crossref","unstructured":"J. Stallkamp et al. 2012. Man vs. computer: Benchmarking machine learning algorithms for traffic sign recognition. Neural Networks (2012).","DOI":"10.1016\/j.neunet.2012.02.016"},{"key":"e_1_3_2_1_51_1","unstructured":"Bob L. Sturm. 2013. The GTZAN dataset: Its contents its faults their effects on evaluation and its future use. (2013). arXiv:1306.1461."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"crossref","unstructured":"E. Tang et al. 2006. An analysis of diversity measures. Mach Learn (2006).","DOI":"10.1007\/s10994-006-9449-2"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"crossref","unstructured":"Siyi Tang et al. 2021. Data Valuation for Medical Imaging Using Shapley Value: Application on A Large-scale Chest X-ray Dataset. Scientific Reports (2021).","DOI":"10.1038\/s41598-021-87762-2"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"crossref","unstructured":"G. Tzanetakis et al. 2002. Musical genre classification of audio signals. IEEE Trans. on Speech and Audio Processing (2002).","DOI":"10.1109\/TSA.2002.800560"},{"key":"e_1_3_2_1_55_1","unstructured":"Udacity. 2016. Self-Driving Car. https:\/\/github.com\/udacity\/self-driving-car"},{"key":"e_1_3_2_1_56_1","volume-title":"Description and Evaluation of a Scalable Learning Classifier System. Evolutionary Intelligence","author":"Urbanowicz Ryan","year":"2015","unstructured":"Ryan Urbanowicz and Jason Moore. 2015. ExSTraCS 2.0: Description and Evaluation of a Scalable Learning Classifier System. Evolutionary Intelligence (2015)."},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"crossref","unstructured":"Kazuya Wakigami Fumio Machida and Tuan Phung-Duc. 2024. Empirical architecture comparison of two-input machine learning systems for vision tasks. Form. Asp. Comput. (2024).","DOI":"10.1145\/3674849"},{"volume-title":"Proc. of CVPR.","author":"Xiaosong","key":"e_1_3_2_1_58_1","unstructured":"Xiaosong Wang et al. 2017. ChestX-Ray8: Hospital-Scale Chest X-Ray Database and Benchmarks on Weakly-Supervised Classification and Localization of Common Thorax Diseases. In Proc. of CVPR."},{"volume-title":"Proc. of ICLR.","author":"Jiaheng","key":"e_1_3_2_1_59_1","unstructured":"Jiaheng Wei et al. 2022. Learning with Noisy Labels Revisited: A Study Using Real-World Human Annotations. In Proc. of ICLR."},{"key":"e_1_3_2_1_60_1","volume-title":"Proc. of AISafety.","author":"Wen Qiang","year":"2024","unstructured":"Qiang Wen, J\u00falio Mendon\u00e7a, Fumio Machida, and Marcus V\u00f6lp. 2024. Enhancing Autonomous Vehicle Safety through N-version Machine Learning Systems. In Proc. of AISafety."},{"volume-title":"Proc. of DSNW.","author":"Hui","key":"e_1_3_2_1_61_1","unstructured":"Hui Xu et al. 2019. NV-DNN: Towards Fault-Tolerant DNN Systems with N-Version Programming. In Proc. of DSNW."},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"crossref","unstructured":"Linfeng Zhang et al. 2019. Be Your Own Teacher: Improve the Performance of Convolutional Neural Networks via Self Distillation. arXiv:1905.08094.","DOI":"10.1109\/ICCV.2019.00381"},{"key":"e_1_3_2_1_63_1","volume-title":"Le","author":"Zoph Barret","year":"2017","unstructured":"Barret Zoph and Quoc V. Le. 2017. Neural Architecture Search with Reinforcement Learning. In Proc. of ICLR."}],"event":{"name":"SAC '25: 40th ACM\/SIGAPP Symposium on Applied Computing","sponsor":["SIGAPP ACM Special Interest Group on Applied Computing"],"location":"Catania International Airport Catania Italy","acronym":"SAC '25"},"container-title":["Proceedings of the 40th ACM\/SIGAPP Symposium on Applied Computing"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3672608.3707978","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3672608.3707978","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:57:37Z","timestamp":1750298257000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3672608.3707978"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,3,31]]},"references-count":61,"alternative-id":["10.1145\/3672608.3707978","10.1145\/3672608"],"URL":"https:\/\/doi.org\/10.1145\/3672608.3707978","relation":{},"subject":[],"published":{"date-parts":[[2025,3,31]]},"assertion":[{"value":"2025-05-14","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}