{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,21]],"date-time":"2025-12-21T10:03:53Z","timestamp":1766311433396,"version":"3.40.3"},"publisher-location":"Cham","reference-count":73,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030501457"},{"type":"electronic","value":"9783030501464"}],"license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.1007\/978-3-030-50146-4_4","type":"book-chapter","created":{"date-parts":[[2020,6,5]],"date-time":"2020-06-05T17:04:54Z","timestamp":1591376694000},"page":"42-55","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":16,"title":["Average Jane, Where Art Thou? \u2013 Recent Avenues in Efficient Machine Learning Under Subjectivity Uncertainty"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2483-5574","authenticated-orcid":false,"given":"Georgios","family":"Rizos","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6478-8699","authenticated-orcid":false,"given":"Bj\u00f6rn W.","family":"Schuller","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2020,6,5]]},"reference":[{"issue":"5","key":"4_CR1","doi-asserted-by":"publisher","first-page":"90","DOI":"10.1145\/3129340","volume":"61","author":"BW Schuller","year":"2018","unstructured":"Schuller, B.W.: Speech emotion recognition: two decades in a nutshell, benchmarks, and ongoing trends. Commun. ACM 61(5), 90\u201399 (2018)","journal-title":"Commun. ACM"},{"issue":"1","key":"4_CR2","doi-asserted-by":"publisher","first-page":"24","DOI":"10.1038\/s41591-018-0316-z","volume":"25","author":"A Esteva","year":"2019","unstructured":"Esteva, A., et al.: A guide to deep learning in healthcare. Nat. Med. 25(1), 24\u201329 (2019)","journal-title":"Nat. Med."},{"key":"4_CR3","unstructured":"Raghu, M., et al.: Direct uncertainty prediction for medical second opinions. In: Proceedings of the International Conference on Machine Learning, pp. 5281\u20135290 (2019)"},{"issue":"3","key":"4_CR4","doi-asserted-by":"publisher","first-page":"341","DOI":"10.1111\/j.1471-6402.1987.tb00908.x","volume":"11","author":"FM Deutsch","year":"1987","unstructured":"Deutsch, F.M., LeBaron, D., Fryer, M.M.: What is in a smile? Psychol. Women Q. 11(3), 341\u2013352 (1987)","journal-title":"Psychol. Women Q."},{"key":"4_CR5","doi-asserted-by":"crossref","unstructured":"Fischer, A.H., Kret, M.E., Broekens, J.: Gender differences in emotion perception and self-reported emotional intelligence: a test of the emotion sensitivity hypothesis. PloS One 13(1) (2018)","DOI":"10.1371\/journal.pone.0190712"},{"issue":"1\u20134","key":"4_CR6","doi-asserted-by":"publisher","first-page":"119","DOI":"10.1080\/00207598108247409","volume":"16","author":"KW McCluskey","year":"1981","unstructured":"McCluskey, K.W., Albas, D.C.: Perception of the emotional content of speech by Canadian and Mexican children, adolescents, and adults. Int. J. Psychol. 16(1\u20134), 119\u2013132 (1981)","journal-title":"Int. J. Psychol."},{"key":"4_CR7","doi-asserted-by":"crossref","unstructured":"Davidson, T., Warmsley, D., Macy, M., Weber, I.: Automated hate speech detection and the problem of offensive language. In: Proceedings of the International AAAI Conference on Web and Social Media (2017)","DOI":"10.1609\/icwsm.v11i1.14955"},{"key":"4_CR8","doi-asserted-by":"crossref","unstructured":"Tschiatschek, S., Singla, A., Gomez Rodriguez, M., Merchant, A., Krause, A.: Fake news detection in social networks via crowd signals. In: Companion Proceedings of the the Web Conference, pp. 517\u2013524 (2018)","DOI":"10.1145\/3184558.3188722"},{"issue":"1\u20132","key":"4_CR9","doi-asserted-by":"publisher","first-page":"59","DOI":"10.1007\/s11263-013-0695-z","volume":"108","author":"G Patterson","year":"2014","unstructured":"Patterson, G., Xu, C., Su, H., Hays, J.: The sun attribute database: beyond categories for deeper scene understanding. Int. J. Comput. Vis. 108(1\u20132), 59\u201381 (2014)","journal-title":"Int. J. Comput. Vis."},{"key":"4_CR10","unstructured":"Smyth, P., Fayyad, U.M., Burl, M.C., Perona, P., Baldi, P.: Inferring ground truth from subjective labelling of venus images. In: Proceedings of Advances in Neural Information Processing Systems, pp. 1085\u20131092 (1995)"},{"issue":"Apr","key":"4_CR11","first-page":"1297","volume":"11","author":"VC Raykar","year":"2010","unstructured":"Raykar, V.C., et al.: Learning from crowds. J. Mach. Learn. Res. 11(Apr), 1297\u20131322 (2010)","journal-title":"J. Mach. Learn. Res."},{"key":"4_CR12","doi-asserted-by":"crossref","unstructured":"Sharmanska, V., Hern\u00e1ndez-Lobato, D., Miguel Hernandez-Lobato, J., Quadrianto, N.: Ambiguity helps: classification with disagreements in crowdsourced annotations. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 2194\u20132202 (2016)","DOI":"10.1109\/CVPR.2016.241"},{"key":"4_CR13","doi-asserted-by":"crossref","unstructured":"Han, J., Zhang, Z., Schmitt, M., Pantic, M., Schuller, B.: From hard to soft: towards more human-like emotion recognition by modelling the perception uncertainty. In: Proceedings of the ACM International Conference on Multimedia, pp. 890\u2013897. ACM (2017)","DOI":"10.1145\/3123266.3123383"},{"key":"4_CR14","doi-asserted-by":"crossref","unstructured":"Rizos, G., Schuller, B.: Modelling sample informativeness for deep affective computing. In: Proceedings of the IEEE International Conference on Acoustics, Speech and Signal Processing, pp. 3482\u20133486. IEEE (2019)","DOI":"10.1109\/ICASSP.2019.8683729"},{"issue":"38","key":"4_CR15","doi-asserted-by":"publisher","first-page":"E7900","DOI":"10.1073\/pnas.1702247114","volume":"114","author":"AS Cowen","year":"2017","unstructured":"Cowen, A.S., Keltner, D.: Self-report captures 27 distinct categories of emotion bridged by continuous gradients. Proc. Natl. Acad. Sci. 114(38), E7900\u2013E7909 (2017)","journal-title":"Proc. Natl. Acad. Sci."},{"key":"4_CR16","doi-asserted-by":"crossref","unstructured":"Ando, A., Kobashikawa, S., Kamiyama, H., Masumura, R., Ijima, Y., Aono, Y.: Soft-target training with ambiguous emotional utterances for DNN-based speech emotion classification. In: Proceedings of the International Conference on Acoustics, Speech and Signal Processing, pp. 4964\u20134968. IEEE (2018)","DOI":"10.1109\/ICASSP.2018.8461299"},{"key":"4_CR17","doi-asserted-by":"crossref","unstructured":"Kazai, G., Kamps, J., Milic-Frayling, N.: Worker types and personality traits in crowdsourcing relevance labels. In: Proceedings of the ACM International Conference on Information and Knowledge Management, pp. 1941\u20131944 (2011)","DOI":"10.1145\/2063576.2063860"},{"key":"4_CR18","doi-asserted-by":"crossref","unstructured":"Tian, Y., Zhu, J.: Learning from crowds in the presence of schools of thought. In: Proceedings of the ACM International Conference on Knowledge Discovery and Data Mining, pp. 226\u2013234 (2012)","DOI":"10.1145\/2339530.2339571"},{"issue":"3","key":"4_CR19","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1007\/s11263-015-0816-y","volume":"115","author":"O Russakovsky","year":"2015","unstructured":"Russakovsky, O., et al.: ImageNet large scale visual recognition challenge. Int. J. Comput. Vis. 115(3), 211\u2013252 (2015)","journal-title":"Int. J. Comput. Vis."},{"key":"4_CR20","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"740","DOI":"10.1007\/978-3-319-10602-1_48","volume-title":"Computer Vision \u2013 ECCV 2014","author":"T-Y Lin","year":"2014","unstructured":"Lin, T.-Y., et al.: Microsoft COCO: common objects in context. In: Fleet, D., Pajdla, T., Schiele, B., Tuytelaars, T. (eds.) ECCV 2014. LNCS, vol. 8693, pp. 740\u2013755. Springer, Cham (2014). https:\/\/doi.org\/10.1007\/978-3-319-10602-1_48"},{"key":"4_CR21","unstructured":"Zhou, B., Lapedriza, A., Xiao, J., Torralba, A., Oliva, A.: Learning deep features for scene recognition using places database. In: Proceedings of Advances in Neural Information Processing Systems, pp. 487\u2013495 (2014)"},{"key":"4_CR22","series-title":"Communications in Computer and Information Science","doi-asserted-by":"publisher","first-page":"667","DOI":"10.1007\/978-981-10-3005-5_55","volume-title":"Pattern Recognition","author":"Y Li","year":"2016","unstructured":"Li, Y., Tao, J., Schuller, B., Shan, S., Jiang, D., Jia, J.: MEC 2016: the multimodal emotion recognition challenge of CCPR 2016. In: Tan, T., Li, X., Chen, X., Zhou, J., Yang, J., Cheng, H. (eds.) CCPR 2016. CCIS, vol. 663, pp. 667\u2013678. Springer, Singapore (2016). https:\/\/doi.org\/10.1007\/978-981-10-3005-5_55"},{"key":"4_CR23","unstructured":"Zhang, C., Chaudhuri, K.: Active learning from weak and strong labelers. In: Proceedings of Advances in Neural Information Processing Systems, pp. 703\u2013711 (2015)"},{"key":"4_CR24","doi-asserted-by":"crossref","unstructured":"Grimm, M., Kroschel, K.: Evaluation of natural emotions using self assessment manikins. In: Proceedings of the IEEE Workshop on Automatic Speech Recognition and Understanding, pp. 381\u2013385. IEEE (2005)","DOI":"10.1109\/ASRU.2005.1566530"},{"key":"4_CR25","doi-asserted-by":"crossref","unstructured":"Schuller, B., Hantke, S., Weninger, F., Han, W., Zhang, Z., Narayanan, S.: Automatic recognition of emotion evoked by general sound events. In: Proceedings of the IEEE International Conference on Acoustics, Speech and Signal Processing, pp. 341\u2013344. IEEE (2012)","DOI":"10.1109\/ICASSP.2012.6287886"},{"key":"4_CR26","unstructured":"Hantke, S., Marchi, E., Schuller, B.: Introducing the weighted trustability evaluator for crowdsourcing exemplified by speaker likability classification. In: Proceedings of the International Conference on Language Resources and Evaluation, pp. 2156\u20132161 (2016)"},{"issue":"2","key":"4_CR27","doi-asserted-by":"publisher","first-page":"92","DOI":"10.1109\/T-AFFC.2011.9","volume":"2","author":"MA Nicolaou","year":"2011","unstructured":"Nicolaou, M.A., Gunes, H., Pantic, M.: Continuous prediction of spontaneous affect from multiple cues and modalities in valence-arousal space. IEEE Trans. Affect. Comput. 2(2), 92\u2013105 (2011)","journal-title":"IEEE Trans. Affect. Comput."},{"issue":"7","key":"4_CR28","doi-asserted-by":"publisher","first-page":"1299","DOI":"10.1109\/TPAMI.2014.16","volume":"36","author":"MA Nicolaou","year":"2014","unstructured":"Nicolaou, M.A., Pavlovic, V., Pantic, M.: Dynamic probabilistic CCA for analysis of affective behavior and fusion of continuous annotations. IEEE Trans. Pattern Anal. Mach. Intell. 36(7), 1299\u20131311 (2014)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"4_CR29","doi-asserted-by":"crossref","unstructured":"Booth, B.M., Mundnich, K., Narayanan, S.S.: A novel method for human bias correction of continuous-time annotations. In: Proceedings of the IEEE International Conference on Acoustics, Speech and Signal Processing, pp. 3091\u20133095. IEEE (2018)","DOI":"10.1109\/ICASSP.2018.8461645"},{"issue":"1","key":"4_CR30","doi-asserted-by":"publisher","first-page":"76","DOI":"10.1109\/TAFFC.2016.2592918","volume":"9","author":"R Gupta","year":"2016","unstructured":"Gupta, R., Audhkhasi, K., Jacokes, Z., Rozga, A., Narayanan, S.S.: Modeling multiple time series annotations as noisy distortions of the ground truth: an expectation-maximization approach. IEEE Trans. Affect. Comput. 9(1), 76\u201389 (2016)","journal-title":"IEEE Trans. Affect. Comput."},{"issue":"2","key":"4_CR31","doi-asserted-by":"publisher","first-page":"97","DOI":"10.1109\/TAFFC.2014.2334294","volume":"6","author":"S Mariooryad","year":"2014","unstructured":"Mariooryad, S., Busso, C.: Correcting time-continuous emotional labels by modeling the reaction lag of evaluators. IEEE Trans. Affect. Comput. 6(2), 97\u2013108 (2014)","journal-title":"IEEE Trans. Affect. Comput."},{"key":"4_CR32","doi-asserted-by":"publisher","first-page":"181","DOI":"10.1016\/j.neucom.2017.12.037","volume":"283","author":"B Bakhtiari","year":"2018","unstructured":"Bakhtiari, B., Yazdi, H.S.: Bayesian filter based on the wisdom of crowds. Neurocomputing 283, 181\u2013195 (2018)","journal-title":"Neurocomputing"},{"issue":"2023\u20132049","key":"4_CR33","first-page":"2","volume":"16","author":"V Vapnik","year":"2015","unstructured":"Vapnik, V., Izmailov, R.: Learning using privileged information: similarity control and knowledge transfer. J. Mach. Learn. Res. 16(2023\u20132049), 2 (2015)","journal-title":"J. Mach. Learn. Res."},{"key":"4_CR34","doi-asserted-by":"crossref","unstructured":"Kim, Y., Provost, E.M.: Leveraging inter-rater agreement for audio-visual emotion recognition. In: Proceedings of the International Conference on Affective Computing and Intelligent Interaction, pp. 553\u2013559. IEEE (2015)","DOI":"10.1109\/ACII.2015.7344624"},{"key":"4_CR35","doi-asserted-by":"crossref","unstructured":"Veit, A., Alldrin, N., Chechik, G., Krasin, I., Gupta, A., Belongie, S.: Learning from noisy large-scale datasets with minimal supervision. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 839\u2013847 (2017)","DOI":"10.1109\/CVPR.2017.696"},{"issue":"1","key":"4_CR36","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/2133366.2133372","volume":"2","author":"F Eyben","year":"2012","unstructured":"Eyben, F., W\u00f6llmer, M., Schuller, B.: A multitask approach to continuous five-dimensional affect sensing in natural speech. ACM Trans. Interact. Intell. Syst. 2(1), 1\u201329 (2012)","journal-title":"ACM Trans. Interact. Intell. Syst."},{"key":"4_CR37","doi-asserted-by":"crossref","unstructured":"Steidl, S., Batliner, A., Schuller, B., Seppi, D.: The hinterland of emotions: facing the open-microphone challenge. In: Proceedings of the International Conference on Affective Computing and Intelligent Interaction and Workshops, pp. 1\u20138. IEEE (2009)","DOI":"10.1109\/ACII.2009.5349499"},{"key":"4_CR38","doi-asserted-by":"crossref","unstructured":"Zhang, Z., Deng, J., Marchi, E., Schuller, B.: Active learning by label uncertainty for acoustic emotion recognition. In: Proceedings of the Annual Conference of the International Speech Communication Association (2013)","DOI":"10.21437\/Interspeech.2013-250"},{"key":"4_CR39","unstructured":"Hinton, G., Vinyals, O., Dean, J.: Distilling the knowledge in a neural network. arXiv preprint arXiv:1503.02531 (2015)"},{"key":"4_CR40","unstructured":"Jin, R., Ghahramani, Z.: Learning with multiple labels. In: Proceedings of Advances in Neural Information Processing Systems, pp. 921\u2013928 (2003)"},{"issue":"2","key":"4_CR41","doi-asserted-by":"publisher","first-page":"402","DOI":"10.1007\/s10618-013-0306-1","volume":"28","author":"PG Ipeirotis","year":"2014","unstructured":"Ipeirotis, P.G., Provost, F., Sheng, V.S., Wang, J.: Repeated labeling using multiple noisy labelers. Data Min. Knowl. Disc. 28(2), 402\u2013441 (2014)","journal-title":"Data Min. Knowl. Disc."},{"key":"4_CR42","doi-asserted-by":"crossref","unstructured":"Kim, Y., Kim, J.: Human-like emotion recognition: multi-label learning from noisy labeled audio-visual expressive speech. In: Proceedings of the IEEE International Conference on Acoustics, Speech and Signal Processing, pp. 5104\u20135108. IEEE (2018)","DOI":"10.1109\/ICASSP.2018.8462011"},{"key":"4_CR43","doi-asserted-by":"crossref","unstructured":"Chou, H.-C., Lee, C.-C.: Every rating matters: joint learning of subjective labels and individual annotators for speech emotion classification. In: Proceedings of the IEEE International Conference on Acoustics, Speech and Signal Processing, pp. 5886\u20135890. IEEE (2019)","DOI":"10.1109\/ICASSP.2019.8682170"},{"key":"4_CR44","doi-asserted-by":"crossref","unstructured":"Zhang, H., Jiang, L., Xu, W.: Multiple noisy label distribution propagation for crowdsourcing. In: Proceedings of the International Joint Conference on Artificial Intelligence, pp. 1473\u20131479. AAAI Press (2019)","DOI":"10.24963\/ijcai.2019\/204"},{"issue":"10","key":"4_CR45","doi-asserted-by":"publisher","first-page":"3172","DOI":"10.1109\/TNNLS.2018.2890148","volume":"30","author":"J Zhang","year":"2019","unstructured":"Zhang, J., Sheng, V.S., Wu, J.: Crowdsourced label aggregation using bilayer collaborative clustering. IEEE Trans. Neural Netw. Learn. Syst. 30(10), 3172\u20133185 (2019)","journal-title":"IEEE Trans. Neural Netw. Learn. Syst."},{"key":"4_CR46","doi-asserted-by":"publisher","first-page":"16 662","DOI":"10.1109\/ACCESS.2020.2964484","volume":"8","author":"Y Liu","year":"2020","unstructured":"Liu, Y., Zhang, W., Yu, Y., et al.: Truth inference with a deep clustering-based aggregation model. IEEE Access 8, 16 662\u201316 675 (2020)","journal-title":"IEEE Access"},{"key":"4_CR47","unstructured":"Yan, Y., et al.: Modeling annotator expertise: learning when everybody knows a bit of something. In: Proceedings of the International Conference on Artificial Intelligence and Statistics, pp. 932\u2013939 (2010)"},{"key":"4_CR48","doi-asserted-by":"crossref","unstructured":"Rodrigues, F., Pereira, F.C.: Deep learning from crowds. In: Proceedings of the AAAI Conference on Artificial Intelligence (2018)","DOI":"10.1609\/aaai.v32i1.11506"},{"key":"4_CR49","doi-asserted-by":"publisher","first-page":"110","DOI":"10.1016\/j.inffus.2018.12.008","volume":"52","author":"P Morales-\u00c1lvarez","year":"2019","unstructured":"Morales-\u00c1lvarez, P., Ruiz, P., Santos-Rodr\u00edguez, R., Molina, R., Katsaggelos, A.K.: Scalable and efficient learning from crowds with gaussian processes. Inf. Fusion 52, 110\u2013127 (2019)","journal-title":"Inf. Fusion"},{"key":"4_CR50","unstructured":"Cohn, T., Specia, L.: Modelling annotator bias with multi-task Gaussian processes: an application to machine translation quality estimation. In: Proceedings of the Annual Meeting of the Association for Computational Linguistics, pp. 32\u201342 (2013)"},{"key":"4_CR51","doi-asserted-by":"crossref","unstructured":"Fayek, H.M., Lech, M., Cavedon, L.: Modeling subjectiveness in emotion recognition with deep neural networks: ensembles vs soft labels. In: Proceedings of the International Joint Conference on Neural Networks, pp. 566\u2013570. IEEE (2016)","DOI":"10.1109\/IJCNN.2016.7727250"},{"key":"4_CR52","doi-asserted-by":"crossref","unstructured":"Guan, M.Y., Gulshan, V., Dai, A.M., Hinton, G.E.: Who said what: modeling individual labelers improves classification. In: Proceedings of the AAAI Conference on Artificial Intelligence (2018)","DOI":"10.1609\/aaai.v32i1.11756"},{"issue":"9","key":"4_CR53","doi-asserted-by":"publisher","first-page":"2883","DOI":"10.1007\/s00521-017-2878-y","volume":"30","author":"Z Shu","year":"2018","unstructured":"Shu, Z., Sheng, V.S., Li, J.: Learning from crowds with active learning and self-healing. Neural Comput. Appl. 30(9), 2883\u20132894 (2018)","journal-title":"Neural Comput. Appl."},{"key":"4_CR54","doi-asserted-by":"crossref","unstructured":"Zhang, Y., Coutinho, E., Zhang, Z., Quan, C., Schuller, B.: Dynamic active learning based on agreement and applied to emotion recognition in spoken interactions. In: Proceedings of the ACM International Conference on Multimodal Interaction, pp. 275\u2013278 (2015)","DOI":"10.1145\/2818346.2820774"},{"key":"4_CR55","unstructured":"Rodrigues, F., Pereira, F., Ribeiro, B.: Gaussian process classification and active learning with multiple annotators. In: Proceedings of the International Conference on Machine Learning, pp. 433\u2013441 (2014)"},{"key":"4_CR56","doi-asserted-by":"crossref","unstructured":"Long, C., Hua, G.: Multi-class multi-annotator active learning with robust Gaussian process for visual recognition. In: Proceedings of the IEEE International Conference on Computer Vision, pp. 2839\u20132847 (2015)","DOI":"10.1109\/ICCV.2015.325"},{"key":"4_CR57","unstructured":"Zhong, J., Tang, K., Zhou, Z.-H.: Active learning from crowds with unsure option. In: Proceedings of the International Joint Conference on Artificial Intelligence (2015)"},{"key":"4_CR58","unstructured":"Calma, A., Sick, B.: Simulation of annotators for active learning: uncertain oracles. In: Proceedings of the ECML PKDD Interactive Adaptive Learning Workshop, p. 49 (2017)"},{"key":"4_CR59","unstructured":"Kendall, A., Gal, Y.: What uncertainties do we need in Bayesian deep learning for computer vision? In: Proceedings of Advances in Neural Information Processing Systems, pp. 5574\u20135584 (2017)"},{"key":"4_CR60","doi-asserted-by":"crossref","unstructured":"Yang, J., Drake, T., Damianou, A., Maarek, Y.: Leveraging crowdsourcing data for deep active learning an application: learning intents in alexa. In: Proceedings of the World Wide Web Conference, pp. 23\u201332 (2018)","DOI":"10.1145\/3178876.3186033"},{"key":"4_CR61","doi-asserted-by":"crossref","unstructured":"Rizos, G., Hemker, K., Schuller, B.: Augment to prevent: short-text data augmentation in deep learning for hate-speech classification. In: Proceedings of the ACM International Conference on Information and Knowledge Management, pp. 991\u20131000 (2019)","DOI":"10.1145\/3357384.3358040"},{"key":"4_CR62","doi-asserted-by":"crossref","unstructured":"Tu, J., Yu, G., Wang, J., Domeniconi, C., Zhang, X.: Attention-aware answers of the crowd. In: Proceedings of the 2020 SIAM International Conference on Data Mining, pp. 451\u2013459. SIAM (2020)","DOI":"10.1137\/1.9781611976236.51"},{"key":"4_CR63","doi-asserted-by":"crossref","unstructured":"Takeoka, K., Dong, Y., Oyamada, M.: Learning with unsure responses. In: Proceedings of the AAAI Conference on Artificial Intelligence. AAAI (2020)","DOI":"10.1609\/aaai.v34i01.5355"},{"key":"4_CR64","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"200","DOI":"10.1007\/978-3-319-06608-0_17","volume-title":"Advances in Knowledge Discovery and Data Mining","author":"Q Hu","year":"2014","unstructured":"Hu, Q., He, Q., Huang, H., Chiew, K., Liu, Z.: Learning from crowds under experts\u2019 supervision. In: Tseng, V.S., Ho, T.B., Zhou, Z.-H., Chen, A.L.P., Kao, H.-Y. (eds.) PAKDD 2014. LNCS (LNAI), vol. 8443, pp. 200\u2013211. Springer, Cham (2014). https:\/\/doi.org\/10.1007\/978-3-319-06608-0_17"},{"key":"4_CR65","doi-asserted-by":"crossref","unstructured":"Liu, M., Jiang, L., Liu, J., Wang, X., Zhu, J., Liu, S.: Improving learning-from-crowds through expert validation. In: Proceedings of the International Joint Conferences on Artificial Intelligence, pp. 2329\u20132336 (2017)","DOI":"10.24963\/ijcai.2017\/324"},{"issue":"1","key":"4_CR66","doi-asserted-by":"publisher","first-page":"235","DOI":"10.1109\/TVCG.2018.2864843","volume":"25","author":"S Liu","year":"2018","unstructured":"Liu, S., Chen, C., Lu, Y., Ouyang, F., Wang, B.: An interactive method to improve crowdsourced annotations. IEEE Trans. Vis. Comput. Graph. 25(1), 235\u2013245 (2018)","journal-title":"IEEE Trans. Vis. Comput. Graph."},{"key":"4_CR67","doi-asserted-by":"crossref","unstructured":"Rodrigues, F., Pereira, F.C.: Beyond expectation: deep joint mean and quantile regression for spatiotemporal problems. IEEE Trans. Neural Netw. Learn. Syst. (2020)","DOI":"10.1109\/TNNLS.2020.2966745"},{"key":"4_CR68","doi-asserted-by":"publisher","first-page":"106816","DOI":"10.1016\/j.csda.2019.106816","volume":"142","author":"Y Kwon","year":"2020","unstructured":"Kwon, Y., Won, J.-H., Kim, B.J., Paik, M.C.: Uncertainty quantification using bayesian neural networks in classification: application to biomedical image segmentation. Comput. Stat. Data Anal. 142, 106816 (2020)","journal-title":"Comput. Stat. Data Anal."},{"key":"4_CR69","doi-asserted-by":"crossref","unstructured":"Hau\u00dfmann, M., Hamprecht, F., Kandemir, M.: Deep active learning with adaptive acquisition. In: Proceedings of the International Joint Conference on Artificial Intelligence, pp. 2470\u20132476. AAAI Press (2019)","DOI":"10.24963\/ijcai.2019\/343"},{"key":"4_CR70","unstructured":"Ghorbani, A., Zou, J.: Data shapley: equitable valuation of data for machine learning. In: Proceedings of the International Conference on Machine Learning, pp. 2242\u20132251 (2019)"},{"key":"4_CR71","doi-asserted-by":"crossref","unstructured":"Shrivastava, A., Gupta, A., Girshick, R.: Training region-based object detectors with online hard example mining. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 761\u2013769 (2016)","DOI":"10.1109\/CVPR.2016.89"},{"key":"4_CR72","doi-asserted-by":"crossref","unstructured":"Pentina, A., Sharmanska, V., Lampert, C.H.: Curriculum learning of multiple tasks. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 5492\u20135500 (2015)","DOI":"10.1109\/CVPR.2015.7299188"},{"key":"4_CR73","doi-asserted-by":"crossref","unstructured":"Kim, B., Kim, H., Kim, K., Kim, S., Kim, J.: Learning not to learn: training deep neural networks with biased data. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 9012\u20139020 (2019)","DOI":"10.1109\/CVPR.2019.00922"}],"container-title":["Communications in Computer and Information Science","Information Processing and Management of Uncertainty in Knowledge-Based Systems"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-50146-4_4","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,10,2]],"date-time":"2023-10-02T00:24:44Z","timestamp":1696206284000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-50146-4_4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"ISBN":["9783030501457","9783030501464"],"references-count":73,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-50146-4_4","relation":{},"ISSN":["1865-0929","1865-0937"],"issn-type":[{"type":"print","value":"1865-0929"},{"type":"electronic","value":"1865-0937"}],"subject":[],"published":{"date-parts":[[2020]]},"assertion":[{"value":"5 June 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"IPMU","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Information Processing and Management of Uncertainty in Knowledge-Based Systems","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Lisbon","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Portugal","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2020","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"15 June 2020","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"19 June 2020","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ipmu2020","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ipmu2020.inesc-id.pt\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"213","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"146","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"27","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"69% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3,2","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"The IPMU 2020 was held virtually due to the coronavirus pandemic.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}