{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,18]],"date-time":"2026-06-18T04:17:11Z","timestamp":1781756231732,"version":"3.54.5"},"reference-count":62,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2021,5,6]],"date-time":"2021-05-06T00:00:00Z","timestamp":1620259200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,5,6]],"date-time":"2021-05-06T00:00:00Z","timestamp":1620259200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/501100002913","name":"Vlaamse Overheid","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100002913","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Data Min Knowl Disc"],"published-print":{"date-parts":[[2021,7]]},"DOI":"10.1007\/s10618-021-00751-x","type":"journal-article","created":{"date-parts":[[2021,5,6]],"date-time":"2021-05-06T09:03:13Z","timestamp":1620291793000},"page":"1435-1469","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":27,"title":["Efficient set-valued prediction in multi-class classification"],"prefix":"10.1007","volume":"35","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9650-9263","authenticated-orcid":false,"given":"Thomas","family":"Mortier","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Marek","family":"Wydmuch","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Krzysztof","family":"Dembczy\u0144ski","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Eyke","family":"H\u00fcllermeier","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Willem","family":"Waegeman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,5,6]]},"reference":[{"key":"751_CR1","unstructured":"Babbar R, Dembczy\u0144ski K (2018) Extreme classification for information retrieval. Tutorial at ECIR 2018, http:\/\/www.cs.put.poznan.pl\/kdembczynski\/xmlc-tutorial-ecir-2018\/xmlc4ir-2018.pdf"},{"key":"751_CR2","doi-asserted-by":"crossref","unstructured":"Babbar R, Sch\u00f6lkopf B (2017) Dismec: Distributed sparse machines for extreme multi-label classification. Proceedings of the Tenth ACM International Conference on Web Search and Data Mining, DOI 10(1145\/3018661):3018741","DOI":"10.1145\/3018661.3018741"},{"key":"751_CR3","doi-asserted-by":"crossref","unstructured":"Balasubramanian V, Ho S, Vovk V (eds) (2014) Conformal Prediction for Reliable Machine Learning: Theory. Morgan Kaufmann, Adaptations and Applications","DOI":"10.1016\/B978-0-12-398537-8.00009-2"},{"key":"751_CR4","unstructured":"Beygelzimer A, Langford J, Lifshits Y, Sorkin G, Strehl A (2009) Conditional probability tree estimation analysis and algorithms. In: Proceedings of the Twenty-Fifth Conference on Uncertainty in Artificial Intelligence, AUAI Press, Arlington, Virginia, United States, UAI \u201909, pp 51\u201358"},{"key":"751_CR5","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/TKDE.2015.2441707","volume":"27","author":"W Bi","year":"2015","unstructured":"Bi W, Kwok J (2015) Bayes-optimal hierarchical multilabel classification. IEEE Trans Knowl Data Eng 27:1\u20131","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"751_CR6","first-page":"581","volume":"9","author":"G Corani","year":"2008","unstructured":"Corani G, Zaffalon M (2008) Learning reliable classifiers from small or incomplete data sets: the naive credal classifier 2. J Mach Learn Res 9:581\u2013621","journal-title":"J Mach Learn Res"},{"key":"751_CR7","doi-asserted-by":"crossref","unstructured":"Corani G, Zaffalon M (2009) Lazy naive credal classifier. In: Proceedings of the 1st ACM SIGKDD Workshop on Knowledge Discovery from Uncertain Data, ACM, pp 30\u201337","DOI":"10.1145\/1610555.1610560"},{"key":"751_CR8","first-page":"2273","volume":"10","author":"JJ Del Coz","year":"2009","unstructured":"Del Coz JJ, D\u00edez J, Bahamonde A (2009) Learning nondeterministic classifiers. J Mach Learn Res 10:2273\u20132293","journal-title":"J Mach Learn Res"},{"key":"751_CR9","unstructured":"Dembczy\u0144ski K, Waegeman W, Cheng W, H\u00fcllermeier E (2012) An analysis of chaining in multi-label classification. In: Proceedings of the European Conference on Artificial Intelligence"},{"key":"751_CR10","doi-asserted-by":"crossref","unstructured":"Dembczy\u0144ski K, Kot\u0142owski W, Waegeman W, Busa-Fekete R, H\u00fcllermeier E (2016) Consistency of probabilistic classifier trees. In: ECML\/PKDD","DOI":"10.1007\/978-3-319-46227-1_32"},{"key":"751_CR11","first-page":"102","volume":"18","author":"C Denis","year":"2017","unstructured":"Denis C, Hebiri M (2017) Confidence sets with expected sizes for multiclass classification. J Mach Learn Res 18:102\u2013128","journal-title":"J Mach Learn Res"},{"key":"751_CR12","first-page":"1192","volume":"80","author":"S Depeweg","year":"2018","unstructured":"Depeweg S, Hern\u00e1ndez-Lobato JM, Doshi-Velez F, Udluft S (2018) Decomposition of uncertainty in Bayesian deep learning for efficient and risk-sensitive learning. ICML, PMLR, Proceedings of Machine Learning Research 80:1192\u20131201","journal-title":"ICML, PMLR, Proceedings of Machine Learning Research"},{"issue":"1","key":"751_CR13","doi-asserted-by":"publisher","first-page":"98","DOI":"10.1007\/s11263-014-0733-5","volume":"111","author":"M Everingham","year":"2006","unstructured":"Everingham M, Eslami ASM, Gool LV, Williams CKI, Winn J, Zisserman A (2006) The pascal visual object classes challenge 2006 (VOC2006) results. Int J comput vision 111(1):98\u2013136","journal-title":"Int J comput vision"},{"key":"751_CR14","unstructured":"Everingham M, Gool LV, Williams CKI, Winn J, Zisserman A (2007) The PASCAL visual object classes challenge 2007 (VOC2007) results"},{"key":"751_CR15","first-page":"1871","volume":"9","author":"RE Fan","year":"2008","unstructured":"Fan RE, Chang KW, Hsieh CJ, Wang XR, Lin CJ (2008) LIBLINEAR: a library for large linear classification. J Mach Learn Res 9:1871\u20131874","journal-title":"J Mach Learn Res"},{"key":"751_CR16","doi-asserted-by":"publisher","first-page":"61","DOI":"10.1186\/s12859-018-2083-8","volume":"19","author":"A Fiannaca","year":"2018","unstructured":"Fiannaca A, Paglia LL, Rosa ML, Bosco GL, Renda G, Rizzo R, Gaglio S, Urso A (2018) Deep learning models for bacteria taxonomic classification of metagenomic data. BMC Bioinformat 19:61\u201376","journal-title":"BMC Bioinformat"},{"key":"751_CR17","unstructured":"Fox J (1997) Applied regression analysis, linear models, and related methods. Sage,"},{"key":"751_CR18","doi-asserted-by":"crossref","unstructured":"Frank E, Kramer S (2004) Ensembles of nested dichotomies for multi-class problems. In: Proceedings of the Twenty-first International Conference on Machine Learning, ACM, New York, NY, USA, ICML \u201904, pp 39","DOI":"10.1145\/1015330.1015363"},{"key":"751_CR19","doi-asserted-by":"crossref","unstructured":"Freitas A (2007) A tutorial on hierarchical classification with applications in bioinformatics. In: Research and Trends in Data Mining Technologies and Applications,, pp 175\u2013208","DOI":"10.4018\/978-1-59904-271-8.ch007"},{"issue":"1","key":"751_CR20","doi-asserted-by":"publisher","first-page":"103","DOI":"10.1023\/B:VISI.0000042993.50813.60","volume":"61","author":"JM Geusebroek","year":"2005","unstructured":"Geusebroek JM, Burghouts G, Smeulders A (2005) The amsterdam library of object images. Int J Comput Vision 61(1):103\u2013112","journal-title":"Int J Comput Vision"},{"key":"751_CR21","unstructured":"Griffin G, Holub A, Perona P (2007) Caltech-256 object category dataset. Tech Rep 7694, California Institute of Technology"},{"key":"751_CR22","unstructured":"H\u00fcllermeier E, Waegeman W (2019) Aleatoric and epistemic uncertainty in machine learning: A tutorial introduction. arXiv:1910.09457"},{"key":"751_CR23","unstructured":"Jansche M (2007) A maximum expected utility framework for binary sequence labeling. In: Association for Computational Linguistics, pp 736\u2013743"},{"key":"751_CR24","unstructured":"Johnson J, Douze M, J\u00e9gou H (2017) Billion-scale similarity search with gpus. arXiv preprint arXiv:1702.08734"},{"key":"751_CR25","unstructured":"Kendall A, Gal Y (2017) What uncertainties do we need in Bayesian deep learning for computer vision? Advances in Neural Information Processing Systems 30: Annual Conference on Neural Information Processing Systems 2017, 4\u20139 December 2017. Long Beach, CA, USA, pp 5580\u20135590"},{"key":"751_CR26","unstructured":"LeCun Y, Cortes C (2010) MNIST handwritten digit database. Tech rep Courant Institute, Google Labs, http:\/\/yann.lecun.com\/exdb\/mnist\/"},{"key":"751_CR27","volume-title":"Caltech101 image dataset","author":"FF Li","year":"2003","unstructured":"Li FF, Andreetto M, Ranzato MA (2003) Caltech101 image dataset. Tech. rep, California Institute of Technology"},{"issue":"5","key":"751_CR28","doi-asserted-by":"publisher","first-page":"760","DOI":"10.1093\/bioinformatics\/btx680","volume":"34","author":"Y Li","year":"2018","unstructured":"Li Y, Wang S, Umarov R, Xie B, Fan M, Li L, Gao X (2018) Deepre: sequence-based enzyme EC number prediction by deep learning. Bioinformatics 34(5):760\u2013769","journal-title":"Bioinformatics"},{"key":"751_CR29","unstructured":"Malkov YA, Yashunin DA (2018) Efficient and robust approximate nearest neighbor search using hierarchical navigable small world graphs. IEEE Transactions on Pattern Analysis and Machine Intelligence pp 1\u20131"},{"issue":"8\u201310","key":"751_CR30","doi-asserted-by":"publisher","first-page":"1537","DOI":"10.1007\/s10994-018-5733-1","volume":"107","author":"V Melnikov","year":"2018","unstructured":"Melnikov V, H\u00fcllermeier E (2018) On the effectiveness of heuristics for learning nested dichotomies: an empirical analysis. Mach Learn 107(8\u201310):1537\u20131560","journal-title":"Mach Learn"},{"issue":"1","key":"751_CR31","doi-asserted-by":"publisher","first-page":"143","DOI":"10.1007\/s10994-016-5593-5","volume":"106","author":"D Mena","year":"2017","unstructured":"Mena D, Monta\u00f1\u00e9s E, Quevedo JR, del Coz JJ (2017) A family of admissible heuristics for A* to perform inference in probabilistic classifier chains. Mach Learn 106(1):143\u2013169","journal-title":"Mach Learn"},{"key":"751_CR32","unstructured":"Mikolov T, Sutskever I, Chen K, Corrado GS, Dean J (2013) Distributed representations of words and phrases and their compositionality. In: Burges CJC, Bottou L, Welling M, Ghahramani Z, Weinberger KQ (eds) Advances in Neural Information Processing Systems 26, Curran Associates, Inc., pp 3111\u20133119"},{"key":"751_CR33","unstructured":"Morin F, Bengio Y (2005) Hierarchical probabilistic neural network language model. In: Proceedings of the Tenth International Workshop on Artificial Intelligence and Statistics, Society for Artificial Intelligence and Statistics, pp 246\u2013252"},{"key":"751_CR34","unstructured":"Naidan B, Boytsov L (2015) Non-metric space library manual. CoRR arXiv:1508.05470"},{"issue":"1","key":"751_CR35","doi-asserted-by":"publisher","first-page":"28","DOI":"10.1007\/s007780200060","volume":"11","author":"G Navarro","year":"2002","unstructured":"Navarro G (2002) Searching in metric spaces by spatial approximation. VLDB J 11(1):28\u201346. https:\/\/doi.org\/10.1007\/s007780200060","journal-title":"VLDB J"},{"key":"751_CR36","doi-asserted-by":"crossref","unstructured":"Nguyen V, Destercke S, Masson M, H\u00fcllermeier E (2018) Reliable multi-class classification based on pairwise epistemic and aleatoric uncertainty. In: IJCAI, ijcai.org, pp 5089\u20135095","DOI":"10.24963\/ijcai.2018\/706"},{"key":"751_CR37","unstructured":"Ofer D (2019) Dbpedia classes. https:\/\/www.kaggle.com\/danofer\/dbpedia-classes\/metadata"},{"key":"751_CR38","doi-asserted-by":"crossref","unstructured":"Oh S (2017) Top-k hierarchical classification. In: AAAI, AAAI Press, pp 2450\u20132456","DOI":"10.1609\/aaai.v31i1.10813"},{"issue":"2","key":"751_CR39","first-page":"315","volume":"18","author":"H Papadopoulos","year":"2008","unstructured":"Papadopoulos H (2008) Inductive conformal prediction: theory and application to neural networks. Tools Artif Intel 18(2):315\u2013330","journal-title":"Tools Artif Intel"},{"key":"751_CR40","doi-asserted-by":"crossref","unstructured":"Partalas I, Kosmopoulos A, Baskiotis N, Arti\u00e8res T, Paliouras G, Gaussier \u00c9, Androutsopoulos I, Amini M, Gallinari P (2015) LSHTC: A benchmark for large-scale text classification. CoRR arXiv:1503.08581","DOI":"10.1145\/2556195.2556208"},{"key":"751_CR41","unstructured":"Paszke A, Gross S, Chintala S, Chanan G, Yang E, DeVito Z, Lin Z, Desmaison A, Antiga L, Lerer A (2017) Automatic differentiation in pytorch. In: NIPS-W"},{"key":"751_CR42","doi-asserted-by":"crossref","unstructured":"Prabhu Y, Varma M (2014) Fastxml: A fast, accurate and stable tree-classifier for extreme multi-label learning. In: KDD","DOI":"10.1145\/2623330.2623651"},{"key":"751_CR43","doi-asserted-by":"crossref","unstructured":"Prabhu Y, Kag A, Harsola S, Agrawal R, Varma M (2018) Parabel: Partitioned label trees for extreme classification with application to dynamic search advertising. In: Proceedings of the International World Wide Web Conference","DOI":"10.1145\/3178876.3185998"},{"key":"751_CR44","first-page":"1177","volume":"20","author":"A Rahimi","year":"2008","unstructured":"Rahimi A, Recht B (2008) Random features for large-scale kernel machines. Adv Neural Inform Process Syst 20:1177\u20131184","journal-title":"Adv Neural Inform Process Syst"},{"key":"751_CR45","unstructured":"Ramaswamy HG, Tewari A, Agarwal S (2015) Consistent algorithms for multiclass classification with a reject option. CoRR arXiv:5050.4137"},{"key":"751_CR46","unstructured":"Rangwala H, Naik A (2017) Large scale hierarchical classification: foundations, algorithms and applications. In: The European Conference on Machine Learning and Principles and Practice of Knowledge Discovery in Databases"},{"key":"751_CR47","unstructured":"RIKEN (2013) Genomic-based 16s ribosomal rna database. https:\/\/metasystems.riken.jp\/grd\/download.html"},{"key":"751_CR48","doi-asserted-by":"publisher","first-page":"16","DOI":"10.1016\/j.ins.2013.07.030","volume":"255","author":"R Senge","year":"2014","unstructured":"Senge R, B\u00f6sner S, Dembczy\u00e9nski K, Haasenritter J, Hirsch O, Donner-Banzhoff N, H\u00fcllermeier E (2014) Reliable classification: Learning classifiers that distinguish aleatoric and epistemic uncertainty. Inf Sci 255:16\u201329","journal-title":"Inf Sci"},{"key":"751_CR49","unstructured":"Shafer G, Vovk V (2008) A tutorial on conformal prediction. J Mach Learn Res 9:371\u2013421"},{"key":"751_CR50","unstructured":"Shrivastava A, Li P (2014) Asymmetric lsh (alsh) for sublinear time maximum inner product search (mips). In: Proceedings of the 27th International Conference on Neural Information Processing Systems - Volume 2, MIT Press, Cambridge, MA, USA, NIPS\u201914, pp 2321\u20132329"},{"key":"751_CR51","unstructured":"Simonyan K, Zisserman A (2014) Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556"},{"issue":"5","key":"751_CR52","doi-asserted-by":"publisher","first-page":"1370","DOI":"10.1007\/s10618-016-0456-z","volume":"30","author":"M Stock","year":"2016","unstructured":"Stock M, Dembczynski K, Baets BD, Waegeman W (2016) Exact and efficient top-k inference for multi-target prediction by querying separable linear relational models. Data Min Knowl Discov 30(5):1370\u20131394. https:\/\/doi.org\/10.1007\/s10618-016-0456-z","journal-title":"Data Min Knowl Discov"},{"key":"751_CR53","unstructured":"Syed S (2016) Submodularity in machine learning. MLRG Summer School, https:\/\/www.stat.ubc.ca\/~saif.syed\/papers\/mlrg_submodularity.pdf"},{"key":"751_CR54","unstructured":"Vondrak J (2019) Optimization of submodular functions tutorial. https:\/\/theory.stanford.edu\/~jvondrak\/data\/submod-tutorial-1.pdf"},{"key":"751_CR55","unstructured":"Vovk V, Gammerman A, Shafer G (2003) Algorithmic Learning in a Random World. Springer-Verlag,"},{"key":"751_CR56","first-page":"3333","volume":"15","author":"W Waegeman","year":"2014","unstructured":"Waegeman W, Dembczy\u0144ski K, Jachnik A, Cheng W, H\u00fcllermeier E (2014) On the Bayes-optimality of F-measure maximizers. J Mach Learn Res 15:3333\u20133388","journal-title":"J Mach Learn Res"},{"key":"751_CR57","doi-asserted-by":"crossref","unstructured":"Yagnik J, Strelow D, Ross DA, sung Lin R (2011) The power of comparative reasoning. In: 2011 International Conference on Computer Vision, pp 2431\u20132438","DOI":"10.1109\/ICCV.2011.6126527"},{"key":"751_CR58","doi-asserted-by":"publisher","first-page":"7447","DOI":"10.1007\/s00500-016-2287-7","volume":"21","author":"G Yang","year":"2017","unstructured":"Yang G, Destercke S, Masson MH (2017a) Cautious classification with nested dichotomies and imprecise probabilities. Soft Comput 21:7447\u20137462","journal-title":"Soft Comput"},{"key":"751_CR59","doi-asserted-by":"publisher","first-page":"4316","DOI":"10.1109\/TCYB.2016.2607237","volume":"47","author":"G Yang","year":"2017","unstructured":"Yang G, Destercke S, Masson MH (2017b) The costs of indeterminacy: how to determine them? IEEE Transact Cybernet 47:4316\u20134327","journal-title":"IEEE Transact Cybernet"},{"key":"751_CR60","unstructured":"Ye N, Chai K, Lee WS, Chieu HL (2012) Optimizing f-measures: a tale of two approaches. In: Proceedings of the International Conference on Machine Learning"},{"key":"751_CR61","doi-asserted-by":"publisher","first-page":"1282","DOI":"10.1016\/j.ijar.2012.06.022","volume":"53","author":"M Zaffalon","year":"2012","unstructured":"Zaffalon M, Giorgio C, Mau\u00e1 DD (2012) Evaluating credal classifiers by utility-discounted predictive accuracy. Int J Approx Reasoning 53:1282\u20131301","journal-title":"Int J Approx Reasoning"},{"key":"751_CR62","unstructured":"Ziyin L, Wang Z, Liang PP, Salakhutdinov R, Morency LP, Ueda M (2019) Deep gamblers: Learning to abstain with portfolio theory. arXiv:1907.00208"}],"container-title":["Data Mining and Knowledge Discovery"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10618-021-00751-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10618-021-00751-x\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10618-021-00751-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,12,26]],"date-time":"2022-12-26T13:23:13Z","timestamp":1672060993000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10618-021-00751-x"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,5,6]]},"references-count":62,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2021,7]]}},"alternative-id":["751"],"URL":"https:\/\/doi.org\/10.1007\/s10618-021-00751-x","relation":{},"ISSN":["1384-5810","1573-756X"],"issn-type":[{"value":"1384-5810","type":"print"},{"value":"1573-756X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,5,6]]},"assertion":[{"value":"15 May 2020","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"12 March 2021","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"6 May 2021","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}