{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,7,20]],"date-time":"2025-07-20T22:52:52Z","timestamp":1753051972635,"version":"3.40.4"},"publisher-location":"Cham","reference-count":42,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030461324"},{"type":"electronic","value":"9783030461331"}],"license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.1007\/978-3-030-46133-1_14","type":"book-chapter","created":{"date-parts":[[2020,4,30]],"date-time":"2020-04-30T07:08:58Z","timestamp":1588230538000},"page":"220-236","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Learning to Calibrate and Rerank Multi-label Predictions"],"prefix":"10.1007","author":[{"given":"Cheng","family":"Li","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Virgil","family":"Pavlu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Javed","family":"Aslam","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Bingyu","family":"Wang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kechen","family":"Qin","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2020,4,30]]},"reference":[{"key":"14_CR1","unstructured":"Belanger, D., McCallum, A.: Structured prediction energy networks. In: Proceedings of the International Conference on Machine Learning (2016)"},{"key":"14_CR2","unstructured":"Brukhim, N., Globerson, A.: Predict and constrain: modeling cardinality in deep structured prediction. arXiv preprint arXiv:1802.04721 (2018)"},{"key":"14_CR3","doi-asserted-by":"crossref","unstructured":"Bucak, S.S., Mallapragada, P.K., Jin, R., Jain, A.K.: Efficient multi-label ranking for multi-class learning: application to object recognition. In: 2009 IEEE 12th International Conference on Computer Vision, pp. 2098\u20132105. IEEE (2009)","DOI":"10.1109\/ICCV.2009.5459460"},{"key":"14_CR4","doi-asserted-by":"crossref","unstructured":"Chen, T., Guestrin, C.: XGBoost: a scalable tree boosting system. In: Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, pp. 785\u2013794. ACM (2016)","DOI":"10.1145\/2939672.2939785"},{"key":"14_CR5","unstructured":"Chen, T., Navr\u00e1til, J., Iyengar, V., Shanmugam, K.: Confidence scoring using whitebox meta-models with linear classifier probes. arXiv preprint arXiv:1805.05396 (2018)"},{"key":"14_CR6","unstructured":"Chen, Y.N., Lin, H.T.: Feature-aware label space dimension reduction for multi-label classification. In: NIPS, pp. 1529\u20131537 (2012)"},{"key":"14_CR7","unstructured":"Cheng, W., H\u00fcllermeier, E., Dembczynski, K.J.: Bayes optimal multilabel classification via probabilistic classifier chains. In: ICML 2010, pp. 279\u2013286 (2010)"},{"issue":"1","key":"14_CR8","doi-asserted-by":"publisher","first-page":"25","DOI":"10.1162\/0891201053630273","volume":"31","author":"M Collins","year":"2005","unstructured":"Collins, M., Koo, T.: Discriminative reranking for natural language parsing. Comput. Linguist. 31(1), 25\u201370 (2005)","journal-title":"Comput. Linguist."},{"key":"14_CR9","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"48","DOI":"10.1007\/978-3-319-10590-1_4","volume-title":"Computer Vision \u2013 ECCV 2014","author":"J Deng","year":"2014","unstructured":"Deng, J., et al.: Large-scale object classification using label relation graphs. In: Fleet, D., Pajdla, T., Schiele, B., Tuytelaars, T. (eds.) ECCV 2014. LNCS, vol. 8689, pp. 48\u201364. Springer, Cham (2014). https:\/\/doi.org\/10.1007\/978-3-319-10590-1_4"},{"issue":"Aug","key":"14_CR10","first-page":"1871","volume":"9","author":"RE Fan","year":"2008","unstructured":"Fan, R.E., Chang, K.W., Hsieh, C.J., Wang, X.R., Lin, C.J.: LIBLINEAR: a library for large linear classification. J. Mach. Learn. Res. 9(Aug), 1871\u20131874 (2008)","journal-title":"J. Mach. Learn. Res."},{"key":"14_CR11","doi-asserted-by":"crossref","unstructured":"Friedman, J.H.: Greedy function approximation: a gradient boosting machine. Ann. Stat. 1189\u20131232 (2001)","DOI":"10.1214\/aos\/1013203451"},{"issue":"2","key":"14_CR12","doi-asserted-by":"publisher","first-page":"133","DOI":"10.1007\/s10994-008-5064-8","volume":"73","author":"J F\u00fcrnkranz","year":"2008","unstructured":"F\u00fcrnkranz, J., H\u00fcllermeier, E., Menc\u00eda, E.L., Brinker, K.: Multilabel classification via calibrated label ranking. Mach. Learn. 73(2), 133\u2013153 (2008)","journal-title":"Mach. Learn."},{"key":"14_CR13","doi-asserted-by":"crossref","unstructured":"Ghamrawi, N., McCallum, A.: Collective multi-label classification. In: Proceedings of the 14th ACM International Conference on Information and Knowledge Management, pp. 195\u2013200. ACM (2005)","DOI":"10.1145\/1099554.1099591"},{"issue":"2","key":"14_CR14","doi-asserted-by":"publisher","first-page":"243","DOI":"10.1111\/j.1467-9868.2007.00587.x","volume":"69","author":"T Gneiting","year":"2007","unstructured":"Gneiting, T., Balabdaoui, F., Raftery, A.E.: Probabilistic forecasts, calibration and sharpness. J. R. Stat. Soc. Ser. B (Stat. Methodol.) 69(2), 243\u2013268 (2007)","journal-title":"J. R. Stat. Soc. Ser. B (Stat. Methodol.)"},{"key":"14_CR15","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"22","DOI":"10.1007\/978-3-540-24775-3_5","volume-title":"Advances in Knowledge Discovery and Data Mining","author":"S Godbole","year":"2004","unstructured":"Godbole, S., Sarawagi, S.: Discriminative methods for multi-labeled classification. In: Dai, H., Srikant, R., Zhang, C. (eds.) PAKDD 2004. LNCS (LNAI), vol. 3056, pp. 22\u201330. Springer, Heidelberg (2004). https:\/\/doi.org\/10.1007\/978-3-540-24775-3_5"},{"key":"14_CR16","unstructured":"Goodfellow, I., et al.: Generative adversarial nets. In: Advances in Neural Information Processing Systems, pp. 2672\u20132680 (2014)"},{"key":"14_CR17","unstructured":"Guo, C., Pleiss, G., Sun, Y., Weinberger, K.Q.: On calibration of modern neural networks. arXiv preprint arXiv:1706.04599 (2017)"},{"key":"14_CR18","unstructured":"Gygli, M., Norouzi, M., Angelova, A.: Deep value networks learn to evaluate and iteratively refine structured outputs. arXiv preprint arXiv:1703.04363 (2017)"},{"key":"14_CR19","unstructured":"Hsu, D., Kakade, S., Langford, J., Zhang, T.: Multi-label prediction via compressed sensing. In: NIPS, vol. 22, pp. 772\u2013780 (2009)"},{"key":"14_CR20","unstructured":"Kuleshov, V., Fenner, N., Ermon, S.: Accurate uncertainties for deep learning using calibrated regression. arXiv preprint arXiv:1807.00263 (2018)"},{"key":"14_CR21","unstructured":"Kuleshov, V., Liang, P.S.: Calibrated structured prediction. In: Advances in Neural Information Processing Systems, pp. 3474\u20133482 (2015)"},{"key":"14_CR22","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"665","DOI":"10.1007\/978-3-642-33460-3_48","volume-title":"Machine Learning and Knowledge Discovery in Databases","author":"A Kumar","year":"2012","unstructured":"Kumar, A., Vembu, S., Menon, A.K., Elkan, C.: Learning and inference in probabilistic classifier chains with beam search. In: Flach, P.A., De Bie, T., Cristianini, N. (eds.) ECML PKDD 2012. LNCS (LNAI), vol. 7523, pp. 665\u2013680. Springer, Heidelberg (2012). https:\/\/doi.org\/10.1007\/978-3-642-33460-3_48"},{"key":"14_CR23","unstructured":"Li, C., Wang, B., Pavlu, V., Aslam, J.A.: Conditional Bernoulli mixtures for multi-label classification. In: Proceedings of the 33rd International Conference on Machine Learning, pp. 2482\u20132491 (2016)"},{"key":"14_CR24","unstructured":"Liu, W., Tsang, I.: On the optimality of classifier chain for multi-label classification. In: Advances in Neural Information Processing Systems, pp. 712\u2013720 (2015)"},{"issue":"3","key":"14_CR25","doi-asserted-by":"publisher","first-page":"1494","DOI":"10.1016\/j.patcog.2013.09.029","volume":"47","author":"E Monta\u00f1es","year":"2014","unstructured":"Monta\u00f1es, E., Senge, R., Barranquero, J., Quevedo, J.R., del Coz, J.J., H\u00fcllermeier, E.: Dependent binary relevance models for multi-label classification. Pattern Recogn. 47(3), 1494\u20131508 (2014)","journal-title":"Pattern Recogn."},{"key":"14_CR26","unstructured":"Nam, J., Menc\u00eda, E.L., Kim, H.J., F\u00fcrnkranz, J.: Maximizing subset accuracy with recurrent neural networks in multi-label classification. In: Advances in Neural Information Processing Systems, pp. 5413\u20135423 (2017)"},{"key":"14_CR27","unstructured":"Park, S.H., F\u00fcrnkranz, J.: Multi-label classification with label constraints. In: ECML PKDD 2008 Workshop on Preference Learning, pp. 157\u2013171 (2008)"},{"issue":"3","key":"14_CR28","first-page":"61","volume":"10","author":"J Platt","year":"1999","unstructured":"Platt, J., et al.: Probabilistic outputs for support vector machines and comparisons to regularized likelihood methods. Adv. Large Margin Classif. 10(3), 61\u201374 (1999)","journal-title":"Adv. Large Margin Classif."},{"key":"14_CR29","doi-asserted-by":"crossref","unstructured":"Qin, K., Li, C., Pavlu, V., Aslam, J.: Adapting RNN sequence prediction model to multi-label set prediction. In: Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, (Long and Short Papers), vol. 1, pp. 3181\u20133190 (2019)","DOI":"10.18653\/v1\/N19-1321"},{"issue":"3","key":"14_CR30","doi-asserted-by":"publisher","first-page":"333","DOI":"10.1007\/s10994-011-5256-5","volume":"85","author":"J Read","year":"2011","unstructured":"Read, J., Pfahringer, B., Holmes, G., Frank, E.: Classifier chains for multi-label classification. Machine. Learn. 85(3), 333\u2013359 (2011)","journal-title":"Machine . Learn."},{"key":"14_CR31","unstructured":"Robertson, T.: Order restricted statistical inference. Technical report (1988)"},{"issue":"2","key":"14_CR32","doi-asserted-by":"publisher","first-page":"465","DOI":"10.1093\/biomet\/70.2.465","volume":"70","author":"S Sasabuchi","year":"1983","unstructured":"Sasabuchi, S., Inutsuka, M., Kulatunga, D.: A multivariate version of isotonic regression. Biometrika 70(2), 465\u2013472 (1983)","journal-title":"Biometrika"},{"key":"14_CR33","unstructured":"Shen, L., Sarkar, A., Och, F.J.: Discriminative reranking for machine translation. In: HLT-NAACL 2004 (2004)"},{"key":"14_CR34","unstructured":"Tsoumakas, G., Dimou, A., Spyromitros, E., Mezaris, V., Kompatsiaris, I., Vlahavas, I.: Correlation-based pruning of stacked binary relevance models for multi-label learning. In: Proceedings of the 1st International Workshop on Learning from Multi-label Data, pp. 101\u2013116 (2009)"},{"key":"14_CR35","doi-asserted-by":"publisher","first-page":"1","DOI":"10.4018\/jdwm.2007070101","volume":"2007","author":"G Tsoumakas","year":"2007","unstructured":"Tsoumakas, G., Katakis, I.: Multi-label classification: an overview. Int. J. Data Warehous. Min. 2007, 1\u201313 (2007)","journal-title":"Int. J. Data Warehous. Min."},{"key":"14_CR36","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"406","DOI":"10.1007\/978-3-540-74958-5_38","volume-title":"Machine Learning: ECML 2007","author":"G Tsoumakas","year":"2007","unstructured":"Tsoumakas, G., Vlahavas, I.: Random k-labelsets: an ensemble method for multilabel classification. In: Kok, J.N., Koronacki, J., Mantaras, R.L., Matwin, S., Mladeni\u010d, D., Skowron, A. (eds.) ECML 2007. LNCS (LNAI), vol. 4701, pp. 406\u2013417. Springer, Heidelberg (2007). https:\/\/doi.org\/10.1007\/978-3-540-74958-5_38"},{"key":"14_CR37","doi-asserted-by":"crossref","unstructured":"Xie, P., Salakhutdinov, R., Mou, L., Xing, E.P.: Deep determinantal point process for large-scale multi-label classification. In: ICCV, pp. 473\u2013482 (2017)","DOI":"10.1109\/ICCV.2017.59"},{"key":"14_CR38","unstructured":"Yen, I.E., Huang, X., Zhong, K., Ravikumar, P., Dhillon, I.S.: PD-Sparse: a primal and dual sparse approach to extreme multiclass and multilabel classification. In: Proceedings of the 33nd International Conference on Machine Learning (2016)"},{"key":"14_CR39","doi-asserted-by":"crossref","unstructured":"Zadrozny, B., Elkan, C.: Transforming classifier scores into accurate multiclass probability estimates. In: KDD, pp. 694\u2013699. ACM (2002)","DOI":"10.1145\/775047.775151"},{"key":"14_CR40","doi-asserted-by":"crossref","unstructured":"Zhang, M.L., Zhang, K.: Multi-label learning by exploiting label dependency. In: KDD, pp. 999\u20131008. ACM (2010)","DOI":"10.1145\/1835804.1835930"},{"issue":"7","key":"14_CR41","doi-asserted-by":"publisher","first-page":"2038","DOI":"10.1016\/j.patcog.2006.12.019","volume":"40","author":"ML Zhang","year":"2007","unstructured":"Zhang, M.L., Zhou, Z.H.: ML-KNN: a lazy learning approach to multi-label learning. Pattern Recogn. 40(7), 2038\u20132048 (2007)","journal-title":"Pattern Recogn."},{"issue":"1\u20132","key":"14_CR42","doi-asserted-by":"publisher","first-page":"69","DOI":"10.1007\/s10994-011-5276-1","volume":"88","author":"T Zhou","year":"2012","unstructured":"Zhou, T., Tao, D., Wu, X.: Compressed labeling on distilled labelsets for multi-label learning. Mach. Learn. 88(1\u20132), 69\u2013126 (2012)","journal-title":"Mach. Learn."}],"container-title":["Lecture Notes in Computer Science","Machine Learning and Knowledge Discovery in Databases"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-46133-1_14","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,29]],"date-time":"2025-04-29T22:05:46Z","timestamp":1745964346000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-46133-1_14"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"ISBN":["9783030461324","9783030461331"],"references-count":42,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-46133-1_14","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2020]]},"assertion":[{"value":"30 April 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECML PKDD","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Joint European Conference on Machine Learning and Knowledge Discovery in Databases","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"W\u00fcrzburg","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Germany","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2019","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"16 September 2019","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"20 September 2019","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecml2019","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/ecmlpkdd2019.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Microsoft CMT","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"733","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"130","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"18% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3.04","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"5.3","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"ECML PKDD Workshops Information: single-blind review, submissions: 200, full papers accepted: 70, short papers accepted: 46","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}