{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,29]],"date-time":"2026-05-29T22:54:35Z","timestamp":1780095275964,"version":"3.54.0"},"reference-count":37,"publisher":"IEEE","license":[{"start":{"date-parts":[[2023,4,18]],"date-time":"2023-04-18T00:00:00Z","timestamp":1681776000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2023,4,18]],"date-time":"2023-04-18T00:00:00Z","timestamp":1681776000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001659","name":"Deutsche Forschungsgemeinschaft","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100001659","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100009318","name":"Helmholtz Association","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100009318","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100005235","name":"Institute for Advanced Study","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100005235","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023,4,18]]},"DOI":"10.1109\/isbi53787.2023.10230497","type":"proceedings-article","created":{"date-parts":[[2023,9,1]],"date-time":"2023-09-01T17:23:07Z","timestamp":1693588987000},"page":"1-6","source":"Crossref","is-referenced-by-count":6,"title":["Approaching Peak Ground Truth"],"prefix":"10.1109","author":[{"given":"Florian","family":"Kofler","sequence":"first","affiliation":[{"name":"Helmholtz Zentrum München,Helmholtz AI,Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Johannes","family":"Wahle","sequence":"additional","affiliation":[{"name":"German Center for Neurodegenerative Diseases (DZNE),Bonn,Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ivan","family":"Ezhov","sequence":"additional","affiliation":[{"name":"Technical University Munich (TUM),Department of Informatics,Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sophia J.","family":"Wagner","sequence":"additional","affiliation":[{"name":"Helmholtz Zentrum München,Helmholtz AI,Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rami","family":"Al-Maskari","sequence":"additional","affiliation":[{"name":"Technical University Munich (TUM),Department of Informatics,Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Emilia","family":"Gryska","sequence":"additional","affiliation":[{"name":"University of Gothenburg,Institute of Clinical Sciences, Sahlgrenska Academy,Department of Radiology,Gothenburg,Sweden"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mihail","family":"Todorov","sequence":"additional","affiliation":[{"name":"Helmholtz Munich,Institute for Tissue Engineering and Regenerative Medicine (iTERM),Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Christina","family":"Bukas","sequence":"additional","affiliation":[{"name":"Helmholtz Zentrum München,Helmholtz AI,Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Felix","family":"Meissen","sequence":"additional","affiliation":[{"name":"Technical University Munich (TUM),Department of Informatics,Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tingying","family":"Peng","sequence":"additional","affiliation":[{"name":"Helmholtz Zentrum München,Helmholtz AI,Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ali","family":"Ert\u00fcrk","sequence":"additional","affiliation":[{"name":"Helmholtz Munich,Institute for Tissue Engineering and Regenerative Medicine (iTERM),Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Daniel","family":"Rueckert","sequence":"additional","affiliation":[{"name":"Technical University Munich (TUM),Department of Informatics,Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rolf","family":"Heckemann","sequence":"additional","affiliation":[{"name":"University of Gothenburg,Department of Medical Radiation Sciences,Gothenburg,Sweden"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jan","family":"Kirschke","sequence":"additional","affiliation":[{"name":"TUM,School of Medicine, Klinikum Rechts der Isar,Department of Diagnostic and Interventional Neuroradiology,Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Claus","family":"Zimmer","sequence":"additional","affiliation":[{"name":"TUM,School of Medicine, Klinikum Rechts der Isar,Department of Diagnostic and Interventional Neuroradiology,Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Benedikt","family":"Wiestler","sequence":"additional","affiliation":[{"name":"TUM,School of Medicine, Klinikum Rechts der Isar,Department of Diagnostic and Interventional Neuroradiology,Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Bjoern","family":"Menze","sequence":"additional","affiliation":[{"name":"University of Zurich,Department of Quantitative Biomedicine,Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Marie","family":"Piraud","sequence":"additional","affiliation":[{"name":"Helmholtz Zentrum München,Helmholtz AI,Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00237"},{"key":"ref35","article-title":"Deep quality estimation: Creating surrogate models for human quality ratings","author":"kofler","year":"2022"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW56347.2022.00532"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1016\/j.cviu.2021.103329"},{"key":"ref15","article-title":"Are we done with imagenet?","author":"beyer","year":"2020"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00275"},{"key":"ref14","first-page":"8634","article-title":"Evaluating machine accuracy on ImageNet","volume":"119","author":"shankar","year":"2020","journal-title":"Proceedings of the 37th International Conference on Machine Learning"},{"key":"ref36","first-page":"13","article-title":"An unsupervised information-theoretic perceptual quality metric","volume":"33","author":"bhardwaj","year":"2020","journal-title":"Advances in neural information processing systems"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1016\/j.euf.2021.05.008"},{"key":"ref30","article-title":"Metrics reloaded-a new recommendation framework for biomedical image analysis validation","author":"reinke","year":"2022","journal-title":"Medical Imaging with Deep Learning (MIDL)"},{"key":"ref11","article-title":"Labeling instructions matter in biomedical image analysis","author":"r\u00e4dsch","year":"2022"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1016\/j.cviu.2018.10.009"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1186\/s12880-015-0068-x"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1038\/s41592-020-01008-z"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2020.101821"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/TMI.2014.2377694"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.369"},{"key":"ref16","first-page":"438","article-title":"Some empirical evidence for annotation noise in a bench-marked dataset","author":"klebanov","year":"2010","journal-title":"Human Language Technologies The 2010 Annual Conference of the North American Chapter of the Association for Computational Linguistics"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1038\/s41593-018-0301-3"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1016\/j.cell.2020.01.030"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2021.102062"},{"key":"ref23","article-title":"Human-in-the-loop machine learning: a state of the art","author":"mosqueira-rey","year":"2022","journal-title":"Artificial Intelligence Review"},{"key":"ref26","first-page":"4804","article-title":"Beyond synthetic noise: Deep learning on controlled noisy labels","author":"jiang","year":"2020","journal-title":"International Conference on Machine Learning"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2013.2292894"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46723-8_71"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-16440-8_48"},{"key":"ref21","first-page":"1533","article-title":"Empirical analysis of aggregation methods for collective annotation","author":"qing","year":"2014","journal-title":"Proceedings of COLING 2014 the 25th International Conference on Computational Linguistics Technical Papers"},{"key":"ref28","article-title":"A probabilistic u-net for segmentation of ambiguous images","volume":"31","author":"kohl","year":"2018","journal-title":"Advances in neural information processing systems"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1016\/j.neucom.2015.12.106"},{"key":"ref29","article-title":"Metrics reloaded: Pitfalls and recommendations for image analysis validation","author":"maier-hein","year":"2022"},{"key":"ref8","article-title":"Are we using appropriate segmentation metrics? identifying correlates of human expert perception for cnn training beyond rolling the dice coefficient","author":"kofler","year":"2021"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-018-07619-7"},{"key":"ref9","article-title":"Common limitations of image processing metrics: A picture story","author":"reinke","year":"2021"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W17-1608"},{"key":"ref3","article-title":"The liver tumor segmentation benchmark (lits)","author":"bilic","year":"2019"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1126\/science.109.2823.103"},{"key":"ref5","article-title":"Identifying the best machine learning algorithms for brain tumor segmentation, progression assessment, and overall survival prediction in the brats challenge","author":"bakas","year":"2018"}],"event":{"name":"2023 IEEE 20th International Symposium on Biomedical Imaging (ISBI)","location":"Cartagena, Colombia","start":{"date-parts":[[2023,4,18]]},"end":{"date-parts":[[2023,4,21]]}},"container-title":["2023 IEEE 20th International Symposium on Biomedical Imaging (ISBI)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/10230311\/10230322\/10230497.pdf?arnumber=10230497","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,9,25]],"date-time":"2023-09-25T17:56:25Z","timestamp":1695664585000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10230497\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,4,18]]},"references-count":37,"URL":"https:\/\/doi.org\/10.1109\/isbi53787.2023.10230497","relation":{},"subject":[],"published":{"date-parts":[[2023,4,18]]}}}