{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,5,7]],"date-time":"2025-05-07T04:15:09Z","timestamp":1746591309024,"version":"3.40.5"},"publisher-location":"Cham","reference-count":20,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030461461"},{"type":"electronic","value":"9783030461478"}],"license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.1007\/978-3-030-46147-8_6","type":"book-chapter","created":{"date-parts":[[2020,5,1]],"date-time":"2020-05-01T02:03:39Z","timestamp":1588298619000},"page":"86-102","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["Cost Sensitive Evaluation of Instance Hardness in Machine Learning"],"prefix":"10.1007","author":[{"given":"Ricardo B. C.","family":"Prud\u00eancio","sequence":"first","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2020,4,30]]},"reference":[{"key":"6_CR1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-84628-172-3","volume-title":"Data Complexity in Pattern Recognition","year":"2006","unstructured":"Basu, M., Ho, T. (eds.): Data Complexity in Pattern Recognition. Springer, London (2006). https:\/\/doi.org\/10.1007\/978-1-84628-172-3"},{"issue":"1","key":"6_CR2","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10994-017-5692-y","volume":"107","author":"P Brazdil","year":"2017","unstructured":"Brazdil, P., Giraud-Carrier, C.: Metalearning and algorithm selection: progress, state of the art and introduction to the 2018 special issue. Mach. Learn. 107(1), 1\u201314 (2017). https:\/\/doi.org\/10.1007\/s10994-017-5692-y","journal-title":"Mach. Learn."},{"key":"6_CR3","doi-asserted-by":"publisher","first-page":"131","DOI":"10.1613\/jair.606","volume":"11","author":"CE Brodley","year":"1999","unstructured":"Brodley, C.E., Friedl, M.A.: Identifying mislabeled training data. J. Artif. Intell. Res. 11, 131\u2013167 (1999)","journal-title":"J. Artif. Intell. Res."},{"issue":"2","key":"6_CR4","doi-asserted-by":"publisher","first-page":"447","DOI":"10.1007\/s00521-016-2458-6","volume":"29","author":"RMO Cruz","year":"2016","unstructured":"Cruz, R.M.O., Sabourin, R., Cavalcanti, G.D.C.: Prototype selection for dynamic classifier and ensemble selection. Neural Comput. Appl. 29(2), 447\u2013457 (2016). https:\/\/doi.org\/10.1007\/s00521-016-2458-6","journal-title":"Neural Comput. Appl."},{"issue":"1","key":"6_CR5","doi-asserted-by":"publisher","first-page":"95","DOI":"10.1007\/s10994-006-8199-5","volume":"65","author":"C Drummond","year":"2006","unstructured":"Drummond, C., Holte, R.C.: Cost curves: an improved method for visualizing classifier performance. Mach. Learn. 65(1), 95\u2013130 (2006). https:\/\/doi.org\/10.1007\/s10994-006-8199-5","journal-title":"Mach. Learn."},{"key":"6_CR6","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"575","DOI":"10.1007\/978-3-540-74958-5_55","volume-title":"Machine Learning: ECML 2007","author":"P Flach","year":"2007","unstructured":"Flach, P., Matsubara, E.T.: A simple lexicographic ranker and probability estimator. In: Kok, J.N., Koronacki, J., Mantaras, R.L., Matwin, S., Mladeni\u010d, D., Skowron, A. (eds.) ECML 2007. LNCS (LNAI), vol. 4701, pp. 575\u2013582. Springer, Heidelberg (2007). https:\/\/doi.org\/10.1007\/978-3-540-74958-5_55"},{"key":"6_CR7","doi-asserted-by":"publisher","first-page":"108","DOI":"10.1016\/j.neucom.2014.10.085","volume":"160","author":"LP Garcia","year":"2015","unstructured":"Garcia, L.P., Carvalho, A.C., Lorena, A.C.: Effect of label noise in the complexity of classification problems. Neurocomputing 160, 108\u2013119 (2015)","journal-title":"Neurocomputing"},{"key":"6_CR8","unstructured":"Hern\u00e1ndez-Orallo, J., Flach, P., Ferri, C.: Brier curves: a new cost-based visualisation of classifier performance. In: 28th International Conference on Machine Learning (2011)"},{"issue":"1","key":"6_CR9","first-page":"2813","volume":"13","author":"J Hern\u00e1ndez-Orallo","year":"2012","unstructured":"Hern\u00e1ndez-Orallo, J., Flach, P., Ferri, C.: A unified view of performance metrics: translating threshold choice into expected classification loss. J. Mach. Learn. Res. 13(1), 2813\u20132869 (2012)","journal-title":"J. Mach. Learn. Res."},{"issue":"1","key":"6_CR10","doi-asserted-by":"publisher","first-page":"71","DOI":"10.1007\/s10994-013-5328-9","volume":"93","author":"J Hern\u00e1ndez-Orallo","year":"2013","unstructured":"Hern\u00e1ndez-Orallo, J., Flach, P., Ferri, C.: ROC curves in cost space. Mach. Learn. 93(1), 71\u201391 (2013). https:\/\/doi.org\/10.1007\/s10994-013-5328-9","journal-title":"Mach. Learn."},{"key":"6_CR11","doi-asserted-by":"publisher","first-page":"27","DOI":"10.1016\/j.knosys.2017.10.026","volume":"140","author":"J Luengo","year":"2018","unstructured":"Luengo, J., Shim, S.O., Alshomrani, S., Altalhi, A., Herrera, F.: CNC-NOS: class noise cleaning by ensemble filtering and noise scoring. Knowl.-Based Syst. 140, 27\u201349 (2018)","journal-title":"Knowl.-Based Syst."},{"key":"6_CR12","unstructured":"Mart\u0131nez-Plumed, F., Prud\u00eancio, R.B., Mart\u0131nez-Us\u00f3, A., Hern\u00e1ndez-Orallo, J.: Making sense of item response theory in machine learning. In: European Conference on Artificial Intelligence, ECAI, pp. 1140\u20131148 (2016)"},{"key":"6_CR13","doi-asserted-by":"crossref","unstructured":"Melville, P., Mooney, R.J.: Diverse ensembles for active learning. In: Proceedings of the 21st International Conference on Machine Learning, p. 74 (2004)","DOI":"10.1145\/1015330.1015385"},{"issue":"3","key":"6_CR14","doi-asserted-by":"publisher","first-page":"1067","DOI":"10.1007\/s10115-016-1003-3","volume":"51","author":"L Mor\u00e1n-Fern\u00e1ndez","year":"2016","unstructured":"Mor\u00e1n-Fern\u00e1ndez, L., Bol\u00f3n-Canedo, V., Alonso-Betanzos, A.: Can classification performance be predicted by complexity measures? A study using microarray data. Knowl. Inf. Syst. 51(3), 1067\u20131090 (2016). https:\/\/doi.org\/10.1007\/s10115-016-1003-3","journal-title":"Knowl. Inf. Syst."},{"issue":"3","key":"6_CR15","doi-asserted-by":"publisher","first-page":"563","DOI":"10.1007\/s10844-015-0368-1","volume":"46","author":"K Napierala","year":"2015","unstructured":"Napierala, K., Stefanowski, J.: Types of minority class examples and their influence on learning classifiers from imbalanced data. J. Intell. Inf. Syst. 46(3), 563\u2013597 (2015). https:\/\/doi.org\/10.1007\/s10844-015-0368-1","journal-title":"J. Intell. Inf. Syst."},{"key":"6_CR16","doi-asserted-by":"publisher","first-page":"120","DOI":"10.1016\/j.neucom.2014.10.086","volume":"160","author":"B Sluban","year":"2015","unstructured":"Sluban, B., Lavrac, N.: Relating ensemble diversity and performance: a study in class noise detection. Neurocomputing 160, 120\u2013131 (2015)","journal-title":"Neurocomputing"},{"issue":"2","key":"6_CR17","doi-asserted-by":"publisher","first-page":"225","DOI":"10.1007\/s10994-013-5422-z","volume":"95","author":"MR Smith","year":"2013","unstructured":"Smith, M.R., Martinez, T., Giraud-Carrier, C.: An instance level analysis of data complexity. Mach. Learn. 95(2), 225\u2013256 (2013). https:\/\/doi.org\/10.1007\/s10994-013-5422-z","journal-title":"Mach. Learn."},{"key":"6_CR18","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"317","DOI":"10.1007\/3-540-44938-8_32","volume-title":"Multiple Classifier Systems","author":"S Verbaeten","year":"2003","unstructured":"Verbaeten, S., Van Assche, A.: Ensemble methods for noise elimination in classification problems. In: Windeatt, T., Roli, F. (eds.) MCS 2003. LNCS, vol. 2709, pp. 317\u2013325. Springer, Heidelberg (2003). https:\/\/doi.org\/10.1007\/3-540-44938-8_32"},{"issue":"3","key":"6_CR19","doi-asserted-by":"publisher","first-page":"207","DOI":"10.1016\/j.inffus.2011.03.007","volume":"13","author":"T Woloszynski","year":"2012","unstructured":"Woloszynski, T., Kurzynski, M., Podsiadlo, P., Stachowiak, G.W.: A measure of competence based on random classification for dynamic ensemble selection. Inf. Fusion 13(3), 207\u2013213 (2012)","journal-title":"Inf. Fusion"},{"key":"6_CR20","doi-asserted-by":"publisher","first-page":"405","DOI":"10.1109\/34.588027","volume":"19","author":"K Woods","year":"1997","unstructured":"Woods, K., Kegelmeyer, W., Bowyer, K.: Combination of multiple classifiers using local accuracy estimates. IEEE Trans. Pattern Anal. Mach. Intell. 19, 405\u2013410 (1997)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."}],"container-title":["Lecture Notes in Computer Science","Machine Learning and Knowledge Discovery in Databases"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-46147-8_6","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,5,6]],"date-time":"2025-05-06T09:28:35Z","timestamp":1746523715000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-46147-8_6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"ISBN":["9783030461461","9783030461478"],"references-count":20,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-46147-8_6","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2020]]},"assertion":[{"value":"30 April 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECML PKDD","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Joint European Conference on Machine Learning and Knowledge Discovery in Databases","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"W\u00fcrzburg","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Germany","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2019","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"16 September 2019","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"20 September 2019","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecml2019","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/ecmlpkdd2019.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Microsoft CMT","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"733","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"130","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"18% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3.04","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"5.3","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"ECML PKDD Workshops Information: single-blind review, submissions: 200, full papers accepted: 70, short papers accepted: 46","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}