{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,27]],"date-time":"2025-11-27T18:08:45Z","timestamp":1764266925648,"version":"3.46.0"},"reference-count":59,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2024,1,8]],"date-time":"2024-01-08T00:00:00Z","timestamp":1704672000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,1,8]],"date-time":"2024-01-08T00:00:00Z","timestamp":1704672000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2024,6]]},"DOI":"10.1007\/s10994-023-06485-9","type":"journal-article","created":{"date-parts":[[2024,1,8]],"date-time":"2024-01-08T13:02:35Z","timestamp":1704718955000},"page":"3849-3868","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Exploiting counter-examples for active learning with partial labels"],"prefix":"10.1007","volume":"113","author":[{"given":"Fei","family":"Zhang","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yunjie","family":"Ye","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Lei","family":"Feng","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhongwen","family":"Rao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jieming","family":"Zhu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Marcus","family":"Kalander","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Chen","family":"Gong","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jianye","family":"Hao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Bo","family":"Han","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,1,8]]},"reference":[{"key":"6485_CR1","unstructured":"Ash, J. T., Zhang, C., Krishnamurthy, A., Langford, J., & Agarwal, A. (2020). Deep batch active learning by diverse, uncertain gradient lower bounds. In International conference on learning representations."},{"key":"6485_CR2","doi-asserted-by":"crossref","unstructured":"Briggs, F., Fern, X. Z., & Raich, R. (2012). Rank-loss support instance machines for miml instance annotation. In Proceedings of the 18th ACM SIGKDD international conference on knowledge discovery and data mining (pp. 534\u2013542).","DOI":"10.1145\/2339530.2339616"},{"key":"6485_CR3","doi-asserted-by":"crossref","unstructured":"Cai, L., Xu, X., Liew, J.H., & Foo, C. S. (2021). Revisiting superpixels for active learning in semantic segmentation with realistic annotation costs. In Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition (pp. 10988\u201310997).","DOI":"10.1109\/CVPR46437.2021.01084"},{"key":"6485_CR4","doi-asserted-by":"crossref","unstructured":"Chakraborty, S. (2020). Asking the right questions to the right users: Active learning with imperfect oracles. In Proceedings of the AAAI conference on artificial intelligence (vol. 34, pp. 3365\u20133372).","DOI":"10.1609\/aaai.v34i04.5738"},{"key":"6485_CR5","doi-asserted-by":"crossref","unstructured":"Chen, J., Schein, A., Ungar, L., & Palmer, M. (2006). An empirical study of the behavior of active learning for word sense disambiguation. In Proceedings of the human language technology conference of the NAACL, main conference (pp. 120\u2013127).","DOI":"10.3115\/1220835.1220851"},{"key":"6485_CR6","first-page":"1501","volume":"12","author":"T Cour","year":"2011","unstructured":"Cour, T., Sapp, B., & Taskar, B. (2011). Learning from partial labels. The Journal of Machine Learning Research, 12, 1501\u20131536.","journal-title":"The Journal of Machine Learning Research"},{"key":"6485_CR7","doi-asserted-by":"crossref","unstructured":"Cour, T., Sapp, B., Jordan, C., & Taskar, B. (2009). Learning from ambiguously labeled images. In 2009 IEEE conference on computer vision and pattern recognition (pp. 919\u2013926).","DOI":"10.1109\/CVPR.2009.5206667"},{"issue":"4","key":"6485_CR8","doi-asserted-by":"publisher","first-page":"349","DOI":"10.1080\/13546780442000222","volume":"11","author":"W De Neys","year":"2005","unstructured":"De Neys, W., Schaeken, W., & d\u2019Ydewalle, G. (2005). Working memory and everyday conditional reasoning: Retrieval and inhibition of stored counterexamples. Thinking & Reasoning, 11(4), 349\u2013381.","journal-title":"Thinking & Reasoning"},{"key":"6485_CR9","doi-asserted-by":"crossref","unstructured":"Donmez, P., & Carbonell, J. G. (2008). Proactive learning: Cost-sensitive active learning with multiple imperfect oracles. In Proceedings of the 17th ACM conference on information and knowledge management (pp. 619\u2013628).","DOI":"10.1145\/1458082.1458165"},{"key":"6485_CR10","doi-asserted-by":"crossref","unstructured":"Du, J., & Ling, C. X. (2010). Active learning with human-like noisy oracle. In 2010 IEEE international conference on data mining (pp. 797\u2013802).","DOI":"10.1109\/ICDM.2010.114"},{"key":"6485_CR11","unstructured":"Fang, M., & Zhu, X. (2012). I don\u2019t know the label: Active learning with blind knowledge. In Proceedings of the 21st international conference on pattern recognition (pp. 2238\u20132241)."},{"key":"6485_CR12","doi-asserted-by":"crossref","unstructured":"Feng, L., & An, B. (2018). Leveraging latent label distributions for partial label learning. In International joint conference on artificial intelligence (pp. 2107\u20132113).","DOI":"10.24963\/ijcai.2018\/291"},{"key":"6485_CR13","unstructured":"Feng, L., Lv, J., Han, B., Xu, M., Niu, G., Geng, X., An, B., & Sugiyama, M. (2020). Provably consistent partial-label learning. In Advances in neural information processing systems."},{"key":"6485_CR14","doi-asserted-by":"crossref","unstructured":"Feng, L., & An, B. (2019a). Partial label learning by semantic difference maximization. In International joint conference on artificial intelligence (pp. 2294\u20132300).","DOI":"10.24963\/ijcai.2019\/318"},{"key":"6485_CR15","doi-asserted-by":"crossref","unstructured":"Feng, L., & An, B. (2019b). Partial label learning with self-guided retraining. In Proceedings of the AAAI conference on artificial intelligence (pp. 3542\u20133549).","DOI":"10.1609\/aaai.v33i01.33013542"},{"key":"6485_CR16","unstructured":"Gal, Y., Islam, R., & Ghahramani, Z. (2017). Deep Bayesian active learning with image data. In International conference on machine learning (pp. 1183\u20131192)."},{"issue":"3","key":"6485_CR17","doi-asserted-by":"publisher","first-page":"967","DOI":"10.1109\/TCYB.2017.2669639","volume":"48","author":"C Gong","year":"2017","unstructured":"Gong, C., Liu, T., Tang, Y., Yang, J., Yang, J., & Tao, D. (2017). A regularization approach for instance-based superset label learning. IEEE Transactions on Cybernetics, 48(3), 967\u2013978.","journal-title":"IEEE Transactions on Cybernetics"},{"key":"6485_CR18","doi-asserted-by":"crossref","unstructured":"Guillaumin, M., Verbeek, J., & Schmid, C. (2010). Multiple instance metric learning from automatically labeled bags of faces. In European conference on computer vision (pp. 634\u2013647).","DOI":"10.1007\/978-3-642-15549-9_46"},{"key":"6485_CR19","doi-asserted-by":"crossref","unstructured":"Haussmann, E., Fenzi, M., Chitta, K., Ivanecky, J., Xu, H., Roy, D., Mittel, A., Koumchatzky, N., Farabet, C., & Alvarez, J. M. (2020). Scalable active learning for object detection. In 2020 IEEE intelligent vehicles symposium (pp. 1430\u20131435).","DOI":"10.1109\/IV47402.2020.9304793"},{"key":"6485_CR20","doi-asserted-by":"crossref","unstructured":"He, K., Zhang, X., Ren, S., & Sun, J. (2016). Deep residual learning for image recognition. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 770\u2013778).","DOI":"10.1109\/CVPR.2016.90"},{"issue":"5","key":"6485_CR21","doi-asserted-by":"publisher","first-page":"419","DOI":"10.3233\/IDA-2006-10503","volume":"10","author":"E H\u00fcllermeier","year":"2006","unstructured":"H\u00fcllermeier, E., & Beringer, J. (2006). Learning from ambiguously labeled examples. Intelligent Data Analysis, 10(5), 419\u2013439.","journal-title":"Intelligent Data Analysis"},{"key":"6485_CR22","unstructured":"Ishida, T., Niu, G., Hu, W., & Sugiyama, M. (2017). Learning from complementary labels. In Advances in neural information processing systems (vol. 30)."},{"issue":"43","key":"6485_CR23","doi-asserted-by":"publisher","first-page":"18243","DOI":"10.1073\/pnas.1012933107","volume":"107","author":"PN Johnson-Laird","year":"2010","unstructured":"Johnson-Laird, P. N. (2010). Mental models and human reasoning. Proceedings of the National Academy of Sciences, 107(43), 18243\u201318250.","journal-title":"Proceedings of the National Academy of Sciences"},{"key":"6485_CR24","doi-asserted-by":"crossref","unstructured":"Joshi, A. J., Porikli, F., & Papanikolopoulos, N. (2009). Multi-class active learning for image classification. In 2009 IEEE conference on computer vision and pattern recognition (pp. 2372\u20132379).","DOI":"10.1109\/CVPRW.2009.5206627"},{"key":"6485_CR25","first-page":"22919","volume":"34","author":"Y-Y Kim","year":"2021","unstructured":"Kim, Y.-Y., Song, K., Jang, J., & Moon, I.-C. (2021). Lada: Look-ahead data acquisition via augmentation for deep active learning. Advances in Neural Information Processing Systems, 34, 22919\u201322930.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"6485_CR26","doi-asserted-by":"crossref","unstructured":"Kim, K., Park, D., Kim, K. I., & Chun, S. Y. (2021). Task-aware variational adversarial active learning. In Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition (pp. 8166\u20138175).","DOI":"10.1109\/CVPR46437.2021.00807"},{"key":"6485_CR27","unstructured":"Kingma, D. P., & Ba, J. (2015). Adam: A method for stochastic optimization. In International conference on learning representations."},{"key":"6485_CR28","unstructured":"Kirsch, A., Amersfoort, J. V., & Gal, Y. (2019). Batchbald: Efficient and diverse batch acquisition for deep bayesian active learning. In Proceedings of the 33rd international conference on neural information processing systems (vol. 32, pp. 7026\u20137037)."},{"key":"6485_CR29","unstructured":"Krizhevsky, A., Hinton, G., et al. (2009). Learning multiple layers of features from tiny images."},{"issue":"11","key":"6485_CR30","doi-asserted-by":"publisher","first-page":"2278","DOI":"10.1109\/5.726791","volume":"86","author":"Y LeCun","year":"1998","unstructured":"LeCun, Y., Bottou, L., Bengio, Y., & Haffner, P. (1998). Gradient-based learning applied to document recognition. Proceedings of the IEEE, 86(11), 2278\u20132324.","journal-title":"Proceedings of the IEEE"},{"key":"6485_CR31","doi-asserted-by":"crossref","unstructured":"Lewis, D. D., & Catlett, J. (1994). Heterogenous uncertainty sampling for supervised learning. In Proceedings of the 11th international conference on international conference on machine learning (pp. 148\u2013156).","DOI":"10.1016\/B978-1-55860-335-6.50026-X"},{"key":"6485_CR32","unstructured":"Liu, L., & Dietterich, T. (2014). Learnability of the superset label learning problem. In International conference on machine learning (pp. 1629\u20131637)."},{"key":"6485_CR33","unstructured":"Liu, L., & Dietterich, T. G. (2012). A conditional multinomial mixture model for superset label learning. In Advances in neu9ral information processing systems (pp. 548\u2013556)."},{"key":"6485_CR34","unstructured":"Luo, W., Schwing, A. G., & Urtasun, R. (2013). Latent structured active learning. In Proceedings of the 26th international conference on neural information processing systems (vol. 1, pp. 728\u2013736)."},{"key":"6485_CR35","unstructured":"Lv, J., Xu, M., Feng, L., Niu, G., Geng, X., & Sugiyama, M. (2020). Progressive identification of true labels for partial-label learning. In International conference on machine learning (pp. 6500\u20136510)."},{"key":"6485_CR36","unstructured":"Netzer, Y., Wang, T., Coates, A., Bissacco, A., Wu, B., & Ng, A. Y. (2011). Reading digits in natural images with unsupervised feature learning."},{"key":"6485_CR37","doi-asserted-by":"crossref","unstructured":"Nguyen, H. T., & Smeulders, A. (2004). Active learning using pre-clustering. In Proceedings of the 21st international conference on machine learning (pp. 79).","DOI":"10.1145\/1015330.1015349"},{"key":"6485_CR38","doi-asserted-by":"crossref","unstructured":"Parvaneh, A., Abbasnejad, E., Teney, D., Haffari, G. R., Van Den\u00a0Hengel, A., & Shi, J. Q. (2022). Active learning by feature mixing. In Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition (pp. 12237\u201312246).","DOI":"10.1109\/CVPR52688.2022.01192"},{"key":"6485_CR39","unstructured":"Perez, L., & Wang, J. (2017). The effectiveness of data augmentation in image classification using deep learning. Preprint at arXiv:abs\/1712.04621"},{"key":"6485_CR40","doi-asserted-by":"crossref","unstructured":"Roth, D., & Small, K. (2006). Margin-based active learning for structured output spaces. In European conference on machine learning (pp. 413\u2013424).","DOI":"10.1007\/11871842_40"},{"key":"6485_CR41","unstructured":"Sener, O., & Savarese, S. (2018). Active learning for convolutional neural networks: A core-set approach. In International conference on learning representations."},{"issue":"3","key":"6485_CR42","first-page":"237","volume":"10","author":"B Settles","year":"1995","unstructured":"Settles, B. (1995). Active learning literature survey. Science, 10(3), 237\u2013304.","journal-title":"Science"},{"key":"6485_CR43","unstructured":"Settles, B., Craven, M., & Ray, S. (2007). Multiple-instance active learning. In Advances in neural information processing systems (vol. 20)."},{"issue":"1","key":"6485_CR44","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s40537-019-0197-0","volume":"6","author":"C Shorten","year":"2019","unstructured":"Shorten, C., & Khoshgoftaar, T. M. (2019). A survey on image data augmentation for deep learning. Journal of Big Data, 6(1), 1\u201348.","journal-title":"Journal of Big Data"},{"key":"6485_CR45","unstructured":"Simonyan, K., & Zisserman, A. (2014) Very deep convolutional networks for large-scale image recognition . Preprint at arXiv:abs\/1409.1556"},{"key":"6485_CR46","doi-asserted-by":"crossref","unstructured":"Sinha, S., Ebrahimi, S., & Darrell, T. (2019). Variational adversarial active learning. In Proceedings of the IEEE\/CVF international conference on computer vision (pp. 5972\u20135981).","DOI":"10.1109\/ICCV.2019.00607"},{"issue":"1","key":"6485_CR47","doi-asserted-by":"publisher","first-page":"107","DOI":"10.3758\/BF03195301","volume":"33","author":"N Verschueren","year":"2005","unstructured":"Verschueren, N., Schaeken, W., & d\u2019Ydewalle, G. (2005). Everyday conditional reasoning: A working memory-dependent tradeoff between counterexample and likelihood use. Memory & Cognition, 33(1), 107\u2013119.","journal-title":"Memory & Cognition"},{"key":"6485_CR48","doi-asserted-by":"crossref","unstructured":"Wang, D. -B., Li, L., & Zhang, M. -L. (2019). Adaptive graph guided disambiguation for partial label learning. In Proceedings of the 25th ACM SIGKDD international conference on knowledge discovery & data mining (pp. 83\u201391).","DOI":"10.1145\/3292500.3330840"},{"key":"6485_CR49","unstructured":"Wen, H., Cui, J., Hang, H., Liu, J., Wang, Y., & Lin, Z. (2021). Leveraged weighted loss for partial label learning. In Proceedings of the 38th international conference on machine learning (vol. 139, pp. 11091\u201311100)."},{"key":"6485_CR50","unstructured":"Xiao, H., Rasul, K., & Vollgraf, R. (2017). Fashion-mnist: a novel image dataset for benchmarking machine learning algorithms. Preprint at arXiv:abs\/1708.07747"},{"key":"6485_CR51","unstructured":"Yan, S., Chaudhuri, K., & Javidi, T. (2016). Active learning from imperfect labelers. In Advances in neural information processing systems (vol. 29)."},{"key":"6485_CR52","doi-asserted-by":"crossref","unstructured":"Yan, Y., & Guo, Y. (2020). Partial label learning with batch label correction. In Proceedings of the AAAI conference on artificial intelligence (vol. 34, pp. 6575\u20136582).","DOI":"10.1609\/aaai.v34i04.6132"},{"issue":"2","key":"6485_CR53","doi-asserted-by":"publisher","first-page":"113","DOI":"10.1007\/s11263-014-0781-x","volume":"113","author":"Y Yang","year":"2015","unstructured":"Yang, Y., Ma, Z., Nie, F., Chang, X., & Hauptmann, A. G. (2015). Multi-class active learning by uncertainty sampling with diversity maximization. International Journal of Computer Vision, 113(2), 113\u2013127.","journal-title":"International Journal of Computer Vision"},{"key":"6485_CR54","unstructured":"Yang, J., Wang, H., Wu, S., Chen, G., & Zhao, J. Active learning with controllable augmentation-induced acquisition. In Proceedings of the 40th international conference on machine learning (accepted)."},{"key":"6485_CR55","doi-asserted-by":"crossref","unstructured":"Yao, Y., Deng, J., Chen, X., Gong, C., Wu, J., & Yang, J. (2020). Deep discriminative cnn with temporal ensembling for ambiguously-labeled image classification. In Proceedings of the AAAI conference on artificial intelligence (vol. 34, pp. 12669\u201312676).","DOI":"10.1609\/aaai.v34i07.6959"},{"key":"6485_CR56","doi-asserted-by":"crossref","unstructured":"Yoo, D., & Kweon, I. S. (2019). Learning loss for active learning. In Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition (pp. 93\u2013102).","DOI":"10.1109\/CVPR.2019.00018"},{"key":"6485_CR57","unstructured":"Yu, F., & Zhang, M. -L. (2016). Maximum margin partial label learning. In Asian conference on machine learning (pp. 96\u2013111)."},{"key":"6485_CR58","doi-asserted-by":"crossref","unstructured":"Zeng, Z., Xiao, S., Jia, K., Chan, T. -H., Gao, S., Xu, D., & Ma, Y. (2013). Learning by associating ambiguously labeled images. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 708\u2013715).","DOI":"10.1109\/CVPR.2013.97"},{"key":"6485_CR59","unstructured":"Zhang, F., Feng, L., Han, B., Liu, T., Niu, G., Qin, T., & Sugiyama, M. (2022). Exploiting class activation value for partial-label learning. In International conference on learning representations."}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-023-06485-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10994-023-06485-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-023-06485-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,27]],"date-time":"2025-11-27T18:04:55Z","timestamp":1764266695000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10994-023-06485-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,1,8]]},"references-count":59,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2024,6]]}},"alternative-id":["6485"],"URL":"https:\/\/doi.org\/10.1007\/s10994-023-06485-9","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"type":"print","value":"0885-6125"},{"type":"electronic","value":"1573-0565"}],"subject":[],"published":{"date-parts":[[2024,1,8]]},"assertion":[{"value":"2 June 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"15 August 2023","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"7 October 2023","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"8 January 2024","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Compliance with ethical standards"}},{"value":"The authors have no competing intereststo declare that are relevant to the content of this article.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"Not applicable.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics approval"}},{"value":"Not applicable.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent to participate"}},{"value":"Not applicable.","order":5,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent for publication"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}