{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,25]],"date-time":"2025-09-25T13:50:33Z","timestamp":1758808233356,"version":"3.40.3"},"publisher-location":"Cham","reference-count":35,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031177200"},{"type":"electronic","value":"9783031177217"}],"license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022]]},"DOI":"10.1007\/978-3-031-17721-7_10","type":"book-chapter","created":{"date-parts":[[2022,9,29]],"date-time":"2022-09-29T23:04:00Z","timestamp":1664492640000},"page":"88-97","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Was that\u00a0so\u00a0Hard? Estimating Human Classification Difficulty"],"prefix":"10.1007","author":[{"given":"Morten Rieger","family":"Hannemose","sequence":"first","affiliation":[]},{"given":"Josefine Vilsb\u00f8ll","family":"Sundgaard","sequence":"additional","affiliation":[]},{"given":"Niels Kvorning","family":"Ternov","sequence":"additional","affiliation":[]},{"given":"Rasmus R.","family":"Paulsen","sequence":"additional","affiliation":[]},{"given":"Anders Nymark","family":"Christensen","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2022,9,30]]},"reference":[{"key":"10_CR1","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"480","DOI":"10.1007\/978-3-642-15745-5_59","volume-title":"Medical Image Computing and Computer-Assisted Intervention \u2013 MICCAI 2010","author":"B Andr\u00e9","year":"2010","unstructured":"Andr\u00e9, B., Vercauteren, T., Buchner, A.M., Shahid, M.W., Wallace, M.B., Ayache, N.: An image retrieval approach to setup difficulty levels in training systems for endomicroscopy diagnosis. In: Jiang, T., Navab, N., Pluim, J.P.W., Viergever, M.A. (eds.) MICCAI 2010. LNCS, vol. 6362, pp. 480\u2013487. Springer, Heidelberg (2010). https:\/\/doi.org\/10.1007\/978-3-642-15745-5_59"},{"key":"10_CR2","doi-asserted-by":"crossref","unstructured":"Bengio, Y., Louradour, J., Collobert, R., Weston, J.: Curriculum learning. In: ICML, pp. 41\u201348 (2009)","DOI":"10.1145\/1553374.1553380"},{"key":"10_CR3","unstructured":"Brockhoff, P., M\u00f8ller, J., Andersen, E., Bacher, P., Christiansen, L.: Introduction to statistics - eNotes (2015)"},{"key":"10_CR4","doi-asserted-by":"publisher","first-page":"606","DOI":"10.1016\/j.ebiom.2019.06.050","volume":"45","author":"D Cha","year":"2019","unstructured":"Cha, D., Pae, C., Seong, S.B., Choi, J.Y., Park, H.J.: Automated diagnosis of ear disease using ensemble deep learning with a big otoendoscopy image database. EBioMedicine 45, 606\u2013614 (2019)","journal-title":"EBioMedicine"},{"key":"10_CR5","doi-asserted-by":"crossref","unstructured":"Codella, N.C., et al.: Skin lesion analysis toward melanoma detection: a challenge at the 2017 international symposium on biomedical imaging (isbi), hosted by the international skin imaging collaboration (ISIC). In: 2018 IEEE 15th international symposium on biomedical imaging (ISBI 2018), pp. 168\u2013172. IEEE (2018)","DOI":"10.1109\/ISBI.2018.8363547"},{"key":"10_CR6","unstructured":"Combalia, M., et al.: Bcn20000: dermoscopic lesions in the wild. arXiv preprint arXiv:1908.02288 (2019)"},{"key":"10_CR7","doi-asserted-by":"crossref","unstructured":"Dagan, I., Engelson, S.P.: Committee-based sampling for training probabilistic classifiers. In: Proceedings of the Twelfth International Conference on Machine Learning, pp. 150\u2013157. Morgan Kaufmann (1995)","DOI":"10.1016\/B978-1-55860-377-6.50027-X"},{"key":"10_CR8","doi-asserted-by":"crossref","unstructured":"Deng, J., Dong, W., Socher, R., Li, L.J., Li, K., Fei-Fei, L.: Imagenet: a large-scale hierarchical image database. In: CVPR, pp. 248\u2013255 (2009)","DOI":"10.1109\/CVPR.2009.5206848"},{"issue":"1","key":"10_CR9","doi-asserted-by":"publisher","first-page":"20","DOI":"10.3758\/BF03196994","volume":"12","author":"R Elio","year":"1984","unstructured":"Elio, R., Anderson, J.R.: The effects of information order and learning mode on schema abstraction. Mem. Cogn. 12(1), 20\u201330 (1984). https:\/\/doi.org\/10.3758\/BF03196994","journal-title":"Mem. Cogn."},{"issue":"1","key":"10_CR10","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1007\/s10994-006-6226-1","volume":"63","author":"P Geurts","year":"2006","unstructured":"Geurts, P., Ernst, D., Wehenkel, L.: Extremely randomized trees. Mach. Learn. 63(1), 3\u201342 (2006)","journal-title":"Mach. Learn."},{"key":"10_CR11","unstructured":"Hacohen, G., Weinshall, D.: On the power of curriculum learning in training deep networks. In: ICML, pp. 2535\u20132544 (2019)"},{"key":"10_CR12","doi-asserted-by":"crossref","unstructured":"He, K., Zhang, X., Ren, S., Sun, J.: Deep residual learning for image recognition. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp. 770\u2013778 (2016)","DOI":"10.1109\/CVPR.2016.90"},{"key":"10_CR13","unstructured":"Kendall, M.G.: Rank correlation methods (1948)"},{"key":"10_CR14","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1007\/978-1-4471-2099-5_1","volume-title":"SIGIR 1994","author":"DD Lewis","year":"1994","unstructured":"Lewis, D.D., Gale, W.A.: A sequential algorithm for training text classifiers. In: Croft, B.W., van Rijsbergen, C.J. (eds.) SIGIR 1994, pp. 3\u201312. Springer, London (1994). https:\/\/doi.org\/10.1007\/978-1-4471-2099-5_1"},{"key":"10_CR15","unstructured":"Li, X., Guo, Y.: Active learning with multi-label SVM classification. In: Twenty-Third International Joint Conference on Artificial Intelligence (2013)"},{"key":"10_CR16","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"484","DOI":"10.1007\/978-3-642-23199-5_36","volume-title":"Machine Learning and Data Mining in Pattern Recognition","author":"D Liu","year":"2011","unstructured":"Liu, D., Xiong, Y., Pulli, K., Shapiro, L.: Estimating image segmentation difficulty. In: Perner, P. (ed.) MLDM 2011. LNCS (LNAI), vol. 6871, pp. 484\u2013495. Springer, Heidelberg (2011). https:\/\/doi.org\/10.1007\/978-3-642-23199-5_36"},{"key":"10_CR17","unstructured":"Van der Maaten, L., Hinton, G.: Visualizing data using t-SNE. J. Mach. Learn. Res. 9(11) (2008)"},{"key":"10_CR18","unstructured":"Musgrave, K., Belongie, S., Lim, S.N.: Pytorch metric learning (2020)"},{"key":"10_CR19","doi-asserted-by":"crossref","unstructured":"Nagle, F., Lavie, N.: Predicting human complexity perception of real-world scenes. Roy. Soc. Open Sci. 7(5), 191487 (2020)","DOI":"10.1098\/rsos.191487"},{"key":"10_CR20","first-page":"2825","volume":"12","author":"F Pedregosa","year":"2011","unstructured":"Pedregosa, F., et al.: Scikit-learn: machine learning in Python. J. Mach. Learn. Res. 12, 2825\u20132830 (2011)","journal-title":"J. Mach. Learn. Res."},{"key":"10_CR21","unstructured":"Ren, P., et al.: A survey of deep active learning. arXiv preprint arXiv:2009.00236 (2020)"},{"issue":"1","key":"10_CR22","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s41235-018-0131-6","volume":"3","author":"BD Roads","year":"2018","unstructured":"Roads, B.D., Xu, B., Robinson, J.K., Tanaka, J.W.: The easy-to-hard training advantage with real-world medical images. Cogn. Res. Principles Implications 3(1), 1\u201313 (2018). https:\/\/doi.org\/10.1186\/s41235-018-0131-6","journal-title":"Cogn. Res. Principles Implications"},{"key":"10_CR23","unstructured":"Sadasivan, V.S., Dasgupta, A.: Statistical measures for defining curriculum scoring function. arXiv preprint arXiv:2103.00147 (2021)"},{"key":"10_CR24","doi-asserted-by":"crossref","unstructured":"Sanakoyeu, A., Tschernezki, V., Buchler, U., Ommer, B.: Divide and conquer the embedding space for metric learning. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 471\u2013480 (2019)","DOI":"10.1109\/CVPR.2019.00056"},{"issue":"6","key":"10_CR25","doi-asserted-by":"publisher","first-page":"1593","DOI":"10.1007\/s00371-020-01922-5","volume":"37","author":"F Scheidegger","year":"2020","unstructured":"Scheidegger, F., Istrate, R., Mariani, G., Benini, L., Bekas, C., Malossi, C.: Efficient image dataset classification difficulty estimation for predicting deep-learning accuracy. Vis. Comput. 37(6), 1593\u20131610 (2020). https:\/\/doi.org\/10.1007\/s00371-020-01922-5","journal-title":"Vis. Comput."},{"key":"10_CR26","unstructured":"Simonyan, K., Zisserman, A.: Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556 (2014)"},{"key":"10_CR27","doi-asserted-by":"crossref","unstructured":"Sundgaard, J.V., et al.: Deep metric learning for otitis media classification. Med. Image Anal. 71, 102034 (2021)","DOI":"10.1016\/j.media.2021.102034"},{"key":"10_CR28","doi-asserted-by":"crossref","unstructured":"Sundgaard, J.V., et al.: Inter-rater reliability of the diagnosis of otitis media based on otoscopic images and wideband tympanometry measurements. Int. J. Pediatr. Otorhinolaryngol. 153, 111034 (2022)","DOI":"10.1016\/j.ijporl.2021.111034"},{"key":"10_CR29","doi-asserted-by":"crossref","unstructured":"Szegedy, C., Vanhoucke, V., Ioffe, S., Shlens, J., Wojna, Z.: Rethinking the inception architecture for computer vision. In: CVPR, pp. 2818\u20132826 (2016)","DOI":"10.1109\/CVPR.2016.308"},{"issue":"1","key":"10_CR30","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/sdata.2018.161","volume":"5","author":"P Tschandl","year":"2018","unstructured":"Tschandl, P., Rosendahl, C., Kittler, H.: The ham10000 dataset, a large collection of multi-source dermatoscopic images of common pigmented skin lesions. Sci. Data 5(1), 1\u20139 (2018)","journal-title":"Sci. Data"},{"key":"10_CR31","doi-asserted-by":"crossref","unstructured":"Tudor Ionescu, R., Alexe, B., Leordeanu, M., Popescu, M., Papadopoulos, D.P., Ferrari, V.: How hard can it be? estimating the difficulty of visual search in an image. In: CVPR, pp. 2157\u20132166 (2016)","DOI":"10.1109\/CVPR.2016.237"},{"issue":"3","key":"10_CR32","doi-asserted-by":"publisher","first-page":"606","DOI":"10.1109\/JSTSP.2011.2139193","volume":"5","author":"D Tuia","year":"2011","unstructured":"Tuia, D., Volpi, M., Copa, L., Kanevski, M., Munoz-Mari, J.: A survey of active learning algorithms for supervised remote sensing image classification. IEEE J. Sel. Top. Signal Process. 5(3), 606\u2013617 (2011)","journal-title":"IEEE J. Sel. Top. Signal Process."},{"key":"10_CR33","doi-asserted-by":"crossref","unstructured":"Vijayanarasimhan, S., Grauman, K.: What\u2019s it going to cost you?: predicting effort vs. informativeness for multi-label image annotations. In: CVPR, pp. 2262\u20132269 (2009)","DOI":"10.1109\/CVPR.2009.5206705"},{"key":"10_CR34","doi-asserted-by":"crossref","unstructured":"Wang, X., Han, X., Huang, W., Dong, D., Scott, M.R.: Multi-similarity loss with general pair weighting for deep metric learning. In: CVPR, pp. 5022\u20135030 (2019)","DOI":"10.1109\/CVPR.2019.00516"},{"issue":"2","key":"10_CR35","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3379504","volume":"53","author":"J Wu","year":"2020","unstructured":"Wu, J., et al.: Multi-label active learning algorithms for image classification: overview and future promise. ACM Comput. Surv. (CSUR) 53(2), 1\u201335 (2020)","journal-title":"ACM Comput. Surv. (CSUR)"}],"container-title":["Lecture Notes in Computer Science","Applications of Medical Artificial Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-17721-7_10","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,9,29]],"date-time":"2022-09-29T23:05:00Z","timestamp":1664492700000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-17721-7_10"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"ISBN":["9783031177200","9783031177217"],"references-count":35,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-17721-7_10","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2022]]},"assertion":[{"value":"30 September 2022","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"AMAI","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Workshop on Applications of Medical AI","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Singapore","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Singapore","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18 September 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18 September 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"amai2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"CMT","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"26","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"17","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"An additional 29 abstracts were submitted to the conference, but were not intended for inclusion in the proceedings.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}