{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,25]],"date-time":"2026-08-25T03:47:47Z","timestamp":1787629667676,"version":"build-2736575974"},"reference-count":43,"publisher":"Springer Science and Business Media LLC","issue":"8","license":[{"start":{"date-parts":[[2022,6,22]],"date-time":"2022-06-22T00:00:00Z","timestamp":1655856000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,6,22]],"date-time":"2022-06-22T00:00:00Z","timestamp":1655856000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100003593","name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico","doi-asserted-by":"publisher","award":["307892\/2020-4"],"award-info":[{"award-number":["307892\/2020-4"]}],"id":[{"id":"10.13039\/501100003593","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002322","name":"Coordena\u00e7\u00e3o de Aperfei\u00e7oamento de Pessoal de N\u00edvel Superior","doi-asserted-by":"publisher","award":["88887.507037\/2020-00"],"award-info":[{"award-number":["88887.507037\/2020-00"]}],"id":[{"id":"10.13039\/501100002322","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000923","name":"Australian Research Council","doi-asserted-by":"crossref","award":["FL140100012"],"award-info":[{"award-number":["FL140100012"]}],"id":[{"id":"10.13039\/501100000923","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100001807","name":"Funda\u00e7\u00e3o de Amparo \u00e0 Pesquisa do Estado de S\u00e3o Paulo","doi-asserted-by":"publisher","award":["2021\/06870-3"],"award-info":[{"award-number":["2021\/06870-3"]}],"id":[{"id":"10.13039\/501100001807","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2022,8]]},"DOI":"10.1007\/s10994-022-06205-9","type":"journal-article","created":{"date-parts":[[2022,6,22]],"date-time":"2022-06-22T21:14:26Z","timestamp":1655932466000},"page":"3085-3123","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":28,"title":["Relating instance hardness to classification performance in a dataset: a visual approach"],"prefix":"10.1007","volume":"111","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9768-7563","authenticated-orcid":false,"given":"Pedro Yuri Arbs","family":"Paiva","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Camila Castro","family":"Moreno","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kate","family":"Smith-Miles","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Maria Gabriela","family":"Valeriano","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ana Carolina","family":"Lorena","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2022,6,22]]},"reference":[{"key":"6205_CR1","doi-asserted-by":"crossref","unstructured":"Arruda, J. L., Prud\u00eancio, R. B., & Lorena, A. C. (2020). Measuring instance hardness using data complexity measures. In Brazilian Conference on Intelligent Systems, Springer, pp 483\u2013497.","DOI":"10.1007\/978-3-030-61380-8_33"},{"issue":"12","key":"6205_CR2","doi-asserted-by":"publisher","DOI":"10.1016\/j.heliyon.2020.e05684","volume":"6","author":"MA Barek","year":"2020","unstructured":"Barek, M. A., Aziz, M. A., & Islam, M. S. (2020). Impact of age, sex, comorbidities and clinical symptoms on the severity of covid-19 cases: A meta-analysis with 55 studies and 10014 cases. Heliyon, 6(12), e05684.","journal-title":"Heliyon"},{"key":"6205_CR3","unstructured":"Bergstra, J., Bardenet, R., Bengio, Y., & K\u00e9gl, B. (2011). Algorithms for hyper-parameter optimization. In Proceedings of the 24th International Conference on Neural Information Processing Systems, Curran Associates Inc., Red Hook, NY, USA, NIPS\u201911, p 2546\u20132554."},{"key":"6205_CR4","unstructured":"Bergstra, J., Yamins, D., & Cox, D. D. (2013). Making a science of model search: Hyperparameter optimization in hundreds of dimensions for vision architectures. In Proc. 30th International Conference on International Conference on Machine Learning - Volume 28, p I\u2013115\u2013I\u2013123."},{"key":"6205_CR5","doi-asserted-by":"publisher","DOI":"10.1016\/j.is.2020.101641","volume":"95","author":"B B\u00f6ken","year":"2021","unstructured":"B\u00f6ken, B. (2021). On the appropriateness of platt scaling in classifier calibration. Information Systems, 95, 101641.","journal-title":"Information Systems"},{"key":"6205_CR6","unstructured":"Corbett-Davies, S., & Goel, S. (2018). The measure and mismeasure of fairness: A critical review of fair machine learning. arXiv preprint arXiv:1808.00023"},{"key":"6205_CR7","first-page":"1","volume":"27","author":"H Edelsbrunner","year":"2010","unstructured":"Edelsbrunner, H. (2010). Alpha shapes\u2013a survey. Tessellations in the Sciences, 27, 1\u201325.","journal-title":"Tessellations in the Sciences"},{"key":"6205_CR8","doi-asserted-by":"crossref","unstructured":"Friedler, S. A., Scheidegger, C., Venkatasubramanian, S., Choudhary, S., Hamilton, E. P., Roth, D. (2019). A comparative study of fairness-enhancing interventions in machine learning. In Proceedings of the conference on fairness, accountability, and transparency, pp 329\u2013338.","DOI":"10.1145\/3287560.3287589"},{"key":"6205_CR9","unstructured":"Gao, S., Ver\u00a0Steeg, G., & Galstyan, A. (2015). Efficient estimation of mutual information for strongly dependent variables. In International Conference on Artificial Intelligence and Statistics (AISTATS), pp 277\u2013286."},{"key":"6205_CR10","doi-asserted-by":"publisher","first-page":"108","DOI":"10.1016\/j.neucom.2014.10.085","volume":"160","author":"LP Garcia","year":"2015","unstructured":"Garcia, L. P., de Carvalho, A. C., & Lorena, A. C. (2015). Effect of label noise in the complexity of classification problems. Neurocomputing, 160, 108\u2013119.","journal-title":"Neurocomputing"},{"key":"6205_CR11","unstructured":"Giraud-Carrier, C., & Provost, F. (2005). Toward a justification of meta-learning: Is the no free lunch theorem a show-stopper. In Proc. ICML-2005 Workshop on Meta-learning, pp 12\u201319."},{"key":"6205_CR12","doi-asserted-by":"crossref","unstructured":"Hajian, S., Bonchi, F., & Castillo, C. (2016). Algorithmic bias: From discrimination discovery to fairness-aware data mining. In Proceedings of the 22nd ACM SIGKDD international conference on knowledge discovery and data mining, pp 2125\u20132126.","DOI":"10.1145\/2939672.2945386"},{"key":"6205_CR13","doi-asserted-by":"crossref","unstructured":"Hillinger, C. (2004). Voting and the cardinal aggregation of judgments. SSRN 548662","DOI":"10.2139\/ssrn.548662"},{"issue":"2","key":"6205_CR14","doi-asserted-by":"publisher","first-page":"309","DOI":"10.1007\/s10618-019-00661-z","volume":"34","author":"S Kandanaarachchi","year":"2020","unstructured":"Kandanaarachchi, S., Mu\u00f1oz, M. A., Hyndman, R. J., & Smith-Miles, K. (2020). On normalization and algorithm selection for unsupervised outlier detection. Data Mining and Knowledge Discovery, 34(2), 309\u2013354.","journal-title":"Data Mining and Knowledge Discovery"},{"issue":"2","key":"6205_CR15","doi-asserted-by":"publisher","first-page":"345","DOI":"10.1016\/j.ijforecast.2016.09.004","volume":"33","author":"Y Kang","year":"2017","unstructured":"Kang, Y., Hyndman, R. J., & Smith-Miles, K. (2017). Visualising forecasting algorithm performance using time series instance spaces. International Journal of Forecasting, 33(2), 345\u2013358.","journal-title":"International Journal of Forecasting"},{"issue":"10","key":"6205_CR16","doi-asserted-by":"publisher","first-page":"13839","DOI":"10.1609\/aaai.v34i10.7192","volume":"34","author":"A Khademi","year":"2020","unstructured":"Khademi, A., & Honavar, V. (2020). Algorithmic bias in recidivism prediction: A causal perspective (student abstract). Proceedings of the AAAI Conference on Artificial Intelligence, 34(10), 13839\u201313840.","journal-title":"Proceedings of the AAAI Conference on Artificial Intelligence"},{"key":"6205_CR17","doi-asserted-by":"crossref","unstructured":"Khan, K., Rehman, S. U., Aziz, K., Fong, S., & Sarasvady, S. (2014). Dbscan: Past, present and future. In The fifth international conference on the applications of digital information and web technologies (ICADIWT 2014), IEEE, pp 232\u2013238.","DOI":"10.1109\/ICADIWT.2014.6814687"},{"key":"6205_CR18","doi-asserted-by":"publisher","first-page":"617","DOI":"10.1007\/s10472-020-09695-2","volume":"89","author":"L Kletzander","year":"2021","unstructured":"Kletzander, L., Musliu, N., & Smith-Miles, K. (2021). Instance space analysis for a personnel scheduling problem. Annals of Mathematics and Artificial Intelligence, 89, 617\u2013637.","journal-title":"Annals of Mathematics and Artificial Intelligence"},{"issue":"2","key":"6205_CR19","doi-asserted-by":"publisher","first-page":"354","DOI":"10.1109\/TKDE.2014.2327034","volume":"27","author":"E Leyva","year":"2014","unstructured":"Leyva, E., Gonz\u00e1lez, A., & P\u00e9rez, R. (2014). A set of complexity measures designed for applying meta-learning to instance selection. IEEE Transactions on Knowledge and Data Engineering, 27(2), 354\u2013367.","journal-title":"IEEE Transactions on Knowledge and Data Engineering"},{"issue":"4","key":"6205_CR20","doi-asserted-by":"publisher","first-page":"1523","DOI":"10.1016\/j.patcog.2014.10.001","volume":"48","author":"E Leyva","year":"2015","unstructured":"Leyva, E., Gonz\u00e1lez, A., & P\u00e9rez, R. (2015). Three new instance selection methods based on local sets: A comparative study with several approaches from a bi-objective perspective. Pattern Recognition, 48(4), 1523\u20131537.","journal-title":"Pattern Recognition"},{"issue":"6","key":"6205_CR21","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3136625","volume":"50","author":"J Li","year":"2017","unstructured":"Li, J., Cheng, K., Wang, S., Morstatter, F., Trevino, R. P., Tang, J., & Liu, H. (2017). Feature selection: A data perspective. ACM Computing Surveys, 50(6), 1\u201345.","journal-title":"ACM Computing Surveys"},{"key":"6205_CR22","unstructured":"Maletic, J. I., & Marcus, A. (2000). Data cleansing: Beyond integrity analysis. In Iq, pp 200\u2013209."},{"issue":"1","key":"6205_CR23","doi-asserted-by":"publisher","first-page":"109","DOI":"10.1007\/s10994-017-5629-5","volume":"107","author":"MA Mu\u00f1oz","year":"2018","unstructured":"Mu\u00f1oz, M. A., Villanova, L., Baatar, D., & Smith-Miles, K. (2018). Instance spaces for machine learning classification. Machine Learning, 107(1), 109\u2013147.","journal-title":"Machine Learning"},{"issue":"4","key":"6205_CR24","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1162\/evco_a_00194","volume":"25","author":"MA Mu\u00f1oz","year":"2017","unstructured":"Mu\u00f1oz, M. A., & Smith-Miles, K. A. (2017). Performance analysis of continuous black-box optimization algorithms via footprints in instance space. Evolutionary computation, 25(4), 529\u2013554.","journal-title":"Evolutionary computation"},{"issue":"2","key":"6205_CR25","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3436893","volume":"15","author":"MA Mu\u00f1oz","year":"2021","unstructured":"Mu\u00f1oz, M. A., Yan, T., Leal, M. R., Smith-Miles, K., Lorena, A. C., Pappa, G. L., & Rodrigues, R. M. (2021). An instance space analysis of regression problems. ACM Transactions on Knowledge Discovery from Data (TKDD), 15(2), 1\u201325.","journal-title":"ACM Transactions on Knowledge Discovery from Data (TKDD)"},{"issue":"3","key":"6205_CR26","first-page":"61","volume":"10","author":"J Platt","year":"1999","unstructured":"Platt, J., et al. (1999). Probabilistic outputs for support vector machines and comparisons to regularized likelihood methods. Advances in large margin classifiers, 10(3), 61\u201374.","journal-title":"Advances in large margin classifiers"},{"key":"6205_CR27","doi-asserted-by":"crossref","unstructured":"Prati, R. C. (2012). Combining feature ranking algorithms through rank aggregation. In: The 2012 International joint conference on neural networks (IJCNN), IEEE, pp 1\u20138.","DOI":"10.1109\/IJCNN.2012.6252467"},{"key":"6205_CR28","first-page":"65","volume-title":"Advances in Computers","author":"JR Rice","year":"1976","unstructured":"Rice, J. R. (1976). The algorithm selection problem. Advances in Computers (Vol. 15, pp. 65\u2013118). Elsevier."},{"key":"6205_CR29","doi-asserted-by":"publisher","DOI":"10.1162\/99608f92.6ed64b30","author":"C Rudin","year":"2020","unstructured":"Rudin, C., Wang, C., & Coker, B. (2020). The age of secrecy and unfairness in recidivism prediction. Harvard Data Science Review. https:\/\/doi.org\/10.1162\/99608f92.6ed64b30.","journal-title":"Harvard Data Science Review"},{"key":"6205_CR30","doi-asserted-by":"crossref","unstructured":"Sani, H. M., Lei, C., & Neagu, D. (2018). Computational complexity analysis of decision tree algorithms. In: International conference on innovative techniques and applications of artificial intelligence, Springer, pp 191\u2013197.","DOI":"10.1007\/978-3-030-04191-5_17"},{"issue":"2","key":"6205_CR31","doi-asserted-by":"publisher","first-page":"225","DOI":"10.1007\/s10994-013-5422-z","volume":"95","author":"MR Smith","year":"2014","unstructured":"Smith, M. R., Martinez, T., & Giraud-Carrier, C. (2014). An instance level analysis of data complexity. Machine Learning, 95(2), 225\u2013256.","journal-title":"Machine Learning"},{"key":"6205_CR32","doi-asserted-by":"publisher","first-page":"102","DOI":"10.1016\/j.cor.2015.04.022","volume":"63","author":"K Smith-Miles","year":"2015","unstructured":"Smith-Miles, K., & Bowly, S. (2015). Generating new test instances by evolving in instance space. Computers & Operations Research, 63, 102\u2013113.","journal-title":"Computers & Operations Research"},{"key":"6205_CR33","doi-asserted-by":"crossref","unstructured":"Smith-Miles, K., & Lopes, L. (2011). Generalising algorithm performance in instance space: A timetabling case study. In: International conference on learning and intelligent optimization, Springer, pp 524\u2013538.","DOI":"10.1007\/978-3-642-25566-3_41"},{"key":"6205_CR34","doi-asserted-by":"crossref","unstructured":"Smith-Miles, K., & Tan, T. T. (2012). Measuring algorithm footprints in instance space. In: 2012 IEEE congress on evolutionary computation, IEEE, pp 1\u20138.","DOI":"10.1109\/CEC.2012.6252992"},{"key":"6205_CR35","doi-asserted-by":"publisher","first-page":"12","DOI":"10.1016\/j.cor.2013.11.015","volume":"45","author":"K Smith-Miles","year":"2014","unstructured":"Smith-Miles, K., Baatar, D., Wreford, B., & Lewis, R. (2014). Towards objective measures of algorithm performance across instance space. Computers and Operations Research, 45, 12\u201324.","journal-title":"Computers and Operations Research"},{"key":"6205_CR36","doi-asserted-by":"publisher","first-page":"105184","DOI":"10.1016\/j.cor.2020.105184","volume":"128","author":"K Smith-Miles","year":"2021","unstructured":"Smith-Miles, K., Christiansen, J., & Mu\u00f1oz, M. A. (2021). Revisiting where are the hard knapsack problems? via instance space analysis. Computers & Operations Research, 128, 105184.","journal-title":"Computers & Operations Research"},{"issue":"1","key":"6205_CR37","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/1456650.1456656","volume":"41","author":"KA Smith-Miles","year":"2009","unstructured":"Smith-Miles, K. A. (2009). Cross-Disciplinary Perspectives on Meta-Learning for Algorithm Selection. ACM Computing Surveys, 41(1), 1\u201325.","journal-title":"ACM Computing Surveys"},{"key":"6205_CR38","unstructured":"Snoek, J., Larochelle, H., & Adams, R. P. (2012). Practical bayesian optimization of machine learning algorithms. In: Proceedings. 25th international conference on neural information processing systems - Volume 2, Curran Associates Inc., Red Hook, NY, USA, p 2951\u20132959."},{"key":"6205_CR39","doi-asserted-by":"crossref","unstructured":"Vanschoren, J. (2019). Meta-learning. In Automated Machine Learning, Springer, pp 35\u201361.","DOI":"10.1007\/978-3-030-05318-5_2"},{"issue":"2","key":"6205_CR40","doi-asserted-by":"publisher","first-page":"77","DOI":"10.1023\/A:1019956318069","volume":"18","author":"R Vilalta","year":"2002","unstructured":"Vilalta, R., & Drissi, Y. (2002). A perspective view and survey of meta-learning. Artificial Intelligence Review, 18(2), 77\u201395.","journal-title":"Artificial Intelligence Review"},{"key":"6205_CR41","first-page":"25","volume-title":"The Supervised Learning No-Free-Lunch Theorems","author":"DH Wolpert","year":"2002","unstructured":"Wolpert, D. H. (2002). The Supervised Learning No-Free-Lunch Theorems (pp. 25\u201342). Springer London."},{"issue":"2","key":"6205_CR42","doi-asserted-by":"publisher","first-page":"e87178","DOI":"10.1371\/journal.pone.0087178","volume":"9","author":"S Yarrow","year":"2014","unstructured":"Yarrow, S., Razak, K. A., Seitz, A. R., & Seri\u00e8s, P. (2014). Detecting and quantifying topography in neural maps. PloS one, 9(2), e87178.","journal-title":"PloS one"},{"issue":"3","key":"6205_CR43","doi-asserted-by":"publisher","first-page":"177","DOI":"10.1007\/s10462-004-0751-8","volume":"22","author":"X Zhu","year":"2004","unstructured":"Zhu, X., & Wu, X. (2004). Class noise vs. attribute noise: A quantitative study. Artificial intelligence review, 22(3), 177\u2013210.","journal-title":"Artificial intelligence review"}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-022-06205-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10994-022-06205-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-022-06205-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,9]],"date-time":"2025-04-09T13:05:35Z","timestamp":1744203935000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10994-022-06205-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,22]]},"references-count":43,"journal-issue":{"issue":"8","published-print":{"date-parts":[[2022,8]]}},"alternative-id":["6205"],"URL":"https:\/\/doi.org\/10.1007\/s10994-022-06205-9","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"value":"0885-6125","type":"print"},{"value":"1573-0565","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,6,22]]},"assertion":[{"value":"5 September 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"3 April 2022","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"26 May 2022","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"22 June 2022","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"The source code of the cited Python packages used in the experiments section is available at the public repositories PyHard () and PyISpace ().","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Code availability"}},{"value":"The anonymized Covid dataset was obtained as part of a partnership between the project \u201cData science for fighting outbreaks, epidemics and pandemics in hospitals\u201d coordinated by researcher A. C. Lorena and the S\u00e3o Jos\u00e9 dos Campos health department.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical approval"}},{"value":"Not Applicable.","order":5,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent to participate"}},{"value":"Not Applicable.","order":6,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent for publication"}}]}}