{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,18]],"date-time":"2025-12-18T09:12:26Z","timestamp":1766049146868,"version":"3.37.3"},"reference-count":40,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2019,5,10]],"date-time":"2019-05-10T00:00:00Z","timestamp":1557446400000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Prog Artif Intell"],"published-print":{"date-parts":[[2019,9]]},"DOI":"10.1007\/s13748-019-00187-x","type":"journal-article","created":{"date-parts":[[2019,5,10]],"date-time":"2019-05-10T13:17:58Z","timestamp":1557494278000},"page":"359-373","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["A framework for evaluation in learning from label proportions"],"prefix":"10.1007","volume":"8","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2257-7164","authenticated-orcid":false,"given":"Jer\u00f3nimo","family":"Hern\u00e1ndez-Gonz\u00e1lez","sequence":"first","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,5,10]]},"reference":[{"key":"187_CR1","doi-asserted-by":"publisher","first-page":"40","DOI":"10.1214\/09-SS054","volume":"4","author":"S Arlot","year":"2010","unstructured":"Arlot, S., Celisse, A.: A survey of cross-validation procedures for model selection. Statist. Surv. 4, 40\u201379 (2010)","journal-title":"Statist. Surv."},{"issue":"1","key":"187_CR2","doi-asserted-by":"publisher","first-page":"5","DOI":"10.1023\/A:1010933404324","volume":"45","author":"L Breiman","year":"2001","unstructured":"Breiman, L.: Random forests. Mach. Learn. 45(1), 5\u201332 (2001)","journal-title":"Mach. Learn."},{"issue":"1","key":"187_CR3","doi-asserted-by":"publisher","first-page":"248","DOI":"10.32614\/RJ-2016-017","volume":"8","author":"B Calvo","year":"2016","unstructured":"Calvo, B., Santaf\u00e9, G.: scmamp: statistical comparison of multiple algorithms in multiple problems. R J. 8(1), 248\u2013256 (2016)","journal-title":"R J."},{"key":"187_CR4","unstructured":"Chang, C.C., Lin, C.J.: LIBSVM: a library for support vector machines (2011). \n                    http:\/\/www.csie.ntu.edu.tw\/~cjlin\/libsvmtools\/datasets\/\n                    \n                  . Accessed 10 Dec 2018"},{"key":"187_CR5","doi-asserted-by":"crossref","unstructured":"Chen, S., Liu, B., Qian, M., Zhang, C.: Kernel k-means based framework for aggregate outputs classification. In: IEEE ICDM Workshops, pp. 356\u2013361 (2009)","DOI":"10.1109\/ICDMW.2009.33"},{"key":"187_CR6","first-page":"1501","volume":"12","author":"T Cour","year":"2011","unstructured":"Cour, T., Sapp, B., Taskar, B.: Learning from partial labels. J. Mach. Learn. Res. 12, 1501\u20131536 (2011)","journal-title":"J. Mach. Learn. Res."},{"key":"187_CR7","doi-asserted-by":"crossref","unstructured":"Davis, J., Goadrich, M.: The relationship between precision-recall and roc curves. In: Proceedings of 23rd ICML, pp. 233\u2013240 (2006)","DOI":"10.1145\/1143844.1143874"},{"key":"187_CR8","first-page":"1","volume":"7","author":"J Dem\u0161ar","year":"2006","unstructured":"Dem\u0161ar, J.: Statistical comparisons of classifiers over multiple data sets. J. Mach. Learn. Res. 7, 1\u201330 (2006)","journal-title":"J. Mach. Learn. Res."},{"issue":"5","key":"187_CR9","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1007\/JHEP05(2017)145","volume":"2017","author":"LM Dery","year":"2017","unstructured":"Dery, L.M., Nachman, B., Rubbo, F., Schwartzman, A.: Weakly supervised classification in high energy physics. J. High Energy Phys. 2017(5), 145 (2017)","journal-title":"J. High Energy Phys."},{"issue":"8","key":"187_CR10","doi-asserted-by":"publisher","first-page":"861","DOI":"10.1016\/j.patrec.2005.10.010","volume":"27","author":"T Fawcett","year":"2006","unstructured":"Fawcett, T.: An introduction to roc analysis. Pattern Recognit. Lett. 27(8), 861\u2013874 (2006)","journal-title":"Pattern Recognit. Lett."},{"key":"187_CR11","doi-asserted-by":"crossref","unstructured":"Feng, L., An, B.: Partial label learning with self-guided retraining. In: Proceedings of 33rd AAAI (2019)","DOI":"10.1609\/aaai.v33i01.33013542"},{"key":"187_CR12","doi-asserted-by":"crossref","unstructured":"Fish, B., Reyzin, L.: On the complexity of learning from label proportions. In: Proceedings of 26th IJCAI, pp. 19\u201325 (2017)","DOI":"10.24963\/ijcai.2017\/232"},{"key":"187_CR13","unstructured":"Frank, A., Asuncion, A.: UCI machine learning repository (2010). \n                    http:\/\/archive.ics.uci.edu\/ml\n                    \n                  . Accessed 10 Dec 2018"},{"key":"187_CR14","unstructured":"Garc\u00eda-Garc\u00eda, D., Williamson, R.C.: Degrees of supervision. In: Proceedings of 25th NIPS, pp. 897\u2013904 (2011)"},{"issue":"3","key":"187_CR15","first-page":"385","volume":"69","author":"DJ Hand","year":"2001","unstructured":"Hand, D.J., Yu, K.: Idiot\u2019s Bayes\u2013not so stupid after all? Int. Stat. Rev. 69(3), 385\u2013398 (2001)","journal-title":"Int. Stat. Rev."},{"key":"187_CR16","doi-asserted-by":"crossref","unstructured":"Hern\u00e1ndez-Gonz\u00e1lez, J.: Evaluation in learning from label proportions: An approximation to the precision-recall curve. In: Proceedings of 18th CAEPIA, pp. 352\u2013362 (2018)","DOI":"10.1007\/978-3-030-00374-6_8"},{"issue":"4","key":"187_CR17","doi-asserted-by":"publisher","first-page":"1056","DOI":"10.1177\/0962280216651098","volume":"27","author":"J Hern\u00e1ndez-Gonz\u00e1lez","year":"2018","unstructured":"Hern\u00e1ndez-Gonz\u00e1lez, J., Inza, I., Crisol-Ortiz, L., Guembe, M.A., I\u00f1arra, M.J., Lozano, J.A.: Fitting the data from embryo implantation prediction: Learning from label proportions. Stat. Methods Med. Res. 27(4), 1056\u20131066 (2018)","journal-title":"Stat. Methods Med. Res."},{"issue":"12","key":"187_CR18","doi-asserted-by":"publisher","first-page":"3425","DOI":"10.1016\/j.patcog.2013.05.002","volume":"46","author":"J Hern\u00e1ndez-Gonz\u00e1lez","year":"2013","unstructured":"Hern\u00e1ndez-Gonz\u00e1lez, J., Inza, I., Lozano, J.A.: Learning bayesian network classifiers from label proportions. Pattern Recognit. 46(12), 3425\u20133440 (2013)","journal-title":"Pattern Recognit."},{"key":"187_CR19","doi-asserted-by":"crossref","unstructured":"Hern\u00e1ndez-Gonz\u00e1lez, J., Inza, I., Lozano, J.A.: Weak supervision and other non-standard classification problems: a taxonomy. Pattern Recognit. Lett. 69, (2016)","DOI":"10.1016\/j.patrec.2015.10.008"},{"issue":"4","key":"187_CR20","doi-asserted-by":"publisher","first-page":"e0175856","DOI":"10.1371\/journal.pone.0175856","volume":"12","author":"D H\u00fcbner","year":"2017","unstructured":"H\u00fcbner, D., Verhoeven, T., Schmid, K., M\u00fcller, K.R., Tangermann, M., Kindermans, P.J.: Learning from label proportions in brain-computer interfaces: online unsupervised learning with guarantees. PloS ONE 12(4), e0175856 (2017)","journal-title":"PloS ONE"},{"key":"187_CR21","unstructured":"Kohavi, R.: A study of cross-validation and bootstrap for accuracy estimation and model selection. In: Proceedings of 14th IJCAI, pp. 1137\u20131143 (1995)"},{"key":"187_CR22","unstructured":"Kohavi, R., Wolpert, D.: Bias plus variance decomposition for zero-one loss functions. In: Proceedings of the 30th ICML, pp. 275\u2013283 (1996)"},{"key":"187_CR23","unstructured":"K\u00fcck, H., de\u00a0Freitas, N.: Learning about individuals from group statistics. In: Proceedings of 21st UAI, pp. 332\u2013339 (2005)"},{"key":"187_CR24","doi-asserted-by":"crossref","unstructured":"Lai, K.T., Yu, F.X., Chen, M.S., Chang, S.F.: Video event detection by inferring temporal instance labels. In: Proceedings of IEEE CVPR, pp. 2251\u20132258 (2014)","DOI":"10.1109\/CVPR.2014.288"},{"key":"187_CR25","volume-title":"Machine Learning","author":"T Mitchell","year":"1997","unstructured":"Mitchell, T.: Machine Learning. McGraw Hill, NY (1997)"},{"key":"187_CR26","doi-asserted-by":"crossref","unstructured":"Musicant, D.R., Christensen, J.M., Olson, J.F.: Supervised learning by training on aggregate outputs. In: Proceedings of 7th IEEE ICDM, pp. 252\u2013261 (2007)","DOI":"10.1109\/ICDM.2007.50"},{"key":"187_CR27","unstructured":"Patrini, G., Nock, R., Rivera, P., Caetano, T.: (Almost) no label no cry. In: Proceedings of NIPS 27, pp. 190\u2013198 (2014)"},{"key":"187_CR28","doi-asserted-by":"crossref","unstructured":"P\u00e9rez-Ortiz, M., Guti\u00e9rrez, P.A., Carbonero-Ruz, M., Herv\u00e1s-Mart\u00ednez, C.: Adapting linear discriminant analysis to the paradigm of learning from label proportions. In: Proceedings of IEEE SSCI, pp. 1\u20137 (2016)","DOI":"10.1109\/SSCI.2016.7850150"},{"issue":"8","key":"187_CR29","doi-asserted-by":"publisher","first-page":"3548","DOI":"10.1109\/TNNLS.2017.2727065","volume":"29","author":"Z Qi","year":"2018","unstructured":"Qi, Z., Meng, F., Tian, Y., Niu, L., Shi, Y., Zhang, P.: Adaboost-llp: a boosting method for learning with label proportions. IEEE Trans. Neural Netw. Learn. Syst. 29(8), 3548\u20133559 (2018)","journal-title":"IEEE Trans. Neural Netw. Learn. Syst."},{"key":"187_CR30","first-page":"2349","volume":"10","author":"N Quadrianto","year":"2009","unstructured":"Quadrianto, N., Smola, A.J., Caetano, T.S., Le, Q.V.: Estimating labels from label proportions. J. Mach. Learn. Res. 10, 2349\u20132374 (2009)","journal-title":"J. Mach. Learn. Res."},{"key":"187_CR31","first-page":"1297","volume":"11","author":"VC Raykar","year":"2010","unstructured":"Raykar, V.C., Yu, S., Zhao, L.H., Valadez, G.H., Florin, C., Bogoni, L., Moy, L.: Learning from crowds. J. Mach. Learn. Res. 11, 1297\u20131322 (2010)","journal-title":"J. Mach. Learn. Res."},{"issue":"3","key":"187_CR32","doi-asserted-by":"publisher","first-page":"569","DOI":"10.1109\/TPAMI.2009.187","volume":"32","author":"JD Rodriguez","year":"2010","unstructured":"Rodriguez, J.D., Perez, A., Lozano, J.A.: Sensitivity analysis of k-fold cross validation in prediction error estimation. IEEE Trans. Pattern Anal. Mach. Intell. 32(3), 569\u2013575 (2010)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"187_CR33","unstructured":"R\u00fcping, S.: SVM classifier estimation from group probabilities. In: Proceedings of 27th ICML, pp. 911\u2013918 (2010)"},{"key":"187_CR34","unstructured":"Sahami, M.: Learning limited dependence Bayesian classifiers. In: Proceedings of 2nd KDD, pp. 335\u2013338 (1996)"},{"issue":"4","key":"187_CR35","doi-asserted-by":"publisher","first-page":"467","DOI":"10.1007\/s10462-015-9433-y","volume":"44","author":"G Santafe","year":"2015","unstructured":"Santafe, G., Inza, I., Lozano, J.A.: Dealing with the evaluation of supervised classification algorithms. Artif. Intell. Rev. 44(4), 467\u2013508 (2015)","journal-title":"Artif. Intell. Rev."},{"key":"187_CR36","doi-asserted-by":"crossref","unstructured":"Shi, Y., Liu, J., Qi, Z.: Inverse convolutional neural networks for learning from label proportions. In: Proceedings of 2018 IEEE\/WIC\/ACM WI, pp. 643\u2013646 (2018)","DOI":"10.1109\/WI.2018.00-21"},{"key":"187_CR37","doi-asserted-by":"crossref","unstructured":"Stolpe, M., Morik, K.: Learning from label proportions by optimizing cluster model selection. In: Proceedings of ECML\/PKDD, vol. 3, pp. 349\u2013364 (2011)","DOI":"10.1007\/978-3-642-23808-6_23"},{"key":"187_CR38","doi-asserted-by":"crossref","unstructured":"Sun, T., Sheldon, D., O\u2019Connor, B.: A probabilistic approach for learning with label proportions applied to the US presidential election. In: IEEE ICDM, pp. 445\u2013454 (2017)","DOI":"10.1109\/ICDM.2017.54"},{"key":"187_CR39","doi-asserted-by":"crossref","unstructured":"Vanschoren, J., van Rijn, J.N., Bischl, B., Torgo, L.: Openml: Networked science in machine learning. SIGKDD Explorations 15(2), 49\u201360 (2013). \n                    https:\/\/www.openml.org\/\n                    \n                  . Accessed 10 Dec 2018","DOI":"10.1145\/2641190.2641198"},{"key":"187_CR40","unstructured":"Yu, F.X., Liu, D., Kumar, S., Jebara, T., Chang, S.: $$\\propto $$SVM for learning with label proportions. In: Proceedings of 30th ICML, pp. 504\u2013512 (2013)"}],"container-title":["Progress in Artificial Intelligence"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s13748-019-00187-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s13748-019-00187-x\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s13748-019-00187-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,5,8]],"date-time":"2020-05-08T19:16:37Z","timestamp":1588965397000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s13748-019-00187-x"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,5,10]]},"references-count":40,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2019,9]]}},"alternative-id":["187"],"URL":"https:\/\/doi.org\/10.1007\/s13748-019-00187-x","relation":{},"ISSN":["2192-6352","2192-6360"],"issn-type":[{"type":"print","value":"2192-6352"},{"type":"electronic","value":"2192-6360"}],"subject":[],"published":{"date-parts":[[2019,5,10]]},"assertion":[{"value":"31 January 2019","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 April 2019","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"10 May 2019","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}