{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,11,17]],"date-time":"2026-11-17T17:15:35Z","timestamp":1794935735639,"version":"4.9.1"},"reference-count":53,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2022,6,18]],"date-time":"2022-06-18T00:00:00Z","timestamp":1655510400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2022,6,18]],"date-time":"2022-06-18T00:00:00Z","timestamp":1655510400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"DOI":"10.13039\/100000070","name":"U.S. Department of Health & Human Services | NIH | National Institute of Biomedical Imaging and Bioengineering","doi-asserted-by":"publisher","award":["R21EB028367"],"award-info":[{"award-number":["R21EB028367"]}],"id":[{"id":"10.13039\/100000070","id-type":"DOI","asserted-by":"publisher"},{"id":"https:\/\/ror.org\/00372qc85","id-type":"ROR","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000070","name":"U.S. Department of Health & Human Services | NIH | National Institute of Biomedical Imaging and Bioengineering","doi-asserted-by":"publisher","award":["R21EB028367"],"award-info":[{"award-number":["R21EB028367"]}],"id":[{"id":"10.13039\/100000070","id-type":"DOI","asserted-by":"publisher"},{"id":"https:\/\/ror.org\/00372qc85","id-type":"ROR","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000070","name":"U.S. Department of Health & Human Services | NIH | National Institute of Biomedical Imaging and Bioengineering","doi-asserted-by":"publisher","award":["R21EB028367"],"award-info":[{"award-number":["R21EB028367"]}],"id":[{"id":"10.13039\/100000070","id-type":"DOI","asserted-by":"publisher"},{"id":"https:\/\/ror.org\/00372qc85","id-type":"ROR","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000070","name":"U.S. Department of Health & Human Services | NIH | National Institute of Biomedical Imaging and Bioengineering","doi-asserted-by":"publisher","award":["R21EB028367"],"award-info":[{"award-number":["R21EB028367"]}],"id":[{"id":"10.13039\/100000070","id-type":"DOI","asserted-by":"publisher"},{"id":"https:\/\/ror.org\/00372qc85","id-type":"ROR","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000070","name":"U.S. Department of Health & Human Services | NIH | National Institute of Biomedical Imaging and Bioengineering","doi-asserted-by":"publisher","award":["R21EB028367"],"award-info":[{"award-number":["R21EB028367"]}],"id":[{"id":"10.13039\/100000070","id-type":"DOI","asserted-by":"publisher"},{"id":"https:\/\/ror.org\/00372qc85","id-type":"ROR","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000070","name":"U.S. Department of Health & Human Services | NIH | National Institute of Biomedical Imaging and Bioengineering","doi-asserted-by":"publisher","award":["R21EB028367"],"award-info":[{"award-number":["R21EB028367"]}],"id":[{"id":"10.13039\/100000070","id-type":"DOI","asserted-by":"publisher"},{"id":"https:\/\/ror.org\/00372qc85","id-type":"ROR","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000070","name":"U.S. Department of Health & Human Services | NIH | National Institute of Biomedical Imaging and Bioengineering","doi-asserted-by":"publisher","award":["R21EB028367"],"award-info":[{"award-number":["R21EB028367"]}],"id":[{"id":"10.13039\/100000070","id-type":"DOI","asserted-by":"publisher"},{"id":"https:\/\/ror.org\/00372qc85","id-type":"ROR","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000070","name":"U.S. Department of Health & Human Services | NIH | National Institute of Biomedical Imaging and Bioengineering","doi-asserted-by":"publisher","award":["R21EB028367"],"award-info":[{"award-number":["R21EB028367"]}],"id":[{"id":"10.13039\/100000070","id-type":"DOI","asserted-by":"publisher"},{"id":"https:\/\/ror.org\/00372qc85","id-type":"ROR","asserted-by":"crossref"}]},{"name":"U.S. Department of Health & Human Services | NIH | National Institute of Biomedical Imaging and Bioengineering","id":[{"id":"https:\/\/ror.org\/00372qc85","id-type":"ROR","asserted-by":"crossref"}]},{"name":"U.S. Department of Health & Human Services | NIH | National Institute of Biomedical Imaging and Bioengineering","id":[{"id":"https:\/\/ror.org\/00372qc85","id-type":"ROR","asserted-by":"crossref"}]},{"name":"U.S. Department of Health & Human Services | NIH | National Institute of Biomedical Imaging and Bioengineering","id":[{"id":"https:\/\/ror.org\/00372qc85","id-type":"ROR","asserted-by":"crossref"}]},{"name":"U.S. Department of Health & Human Services | NIH | National Institute of Biomedical Imaging and Bioengineering","id":[{"id":"https:\/\/ror.org\/00372qc85","id-type":"ROR","asserted-by":"crossref"}]},{"name":"U.S. Department of Health & Human Services | NIH | National Institute of Biomedical Imaging and Bioengineering","id":[{"id":"https:\/\/ror.org\/00372qc85","id-type":"ROR","asserted-by":"crossref"}]},{"name":"U.S. Department of Health & Human Services | NIH | National Institute of Biomedical Imaging and Bioengineering","id":[{"id":"https:\/\/ror.org\/00372qc85","id-type":"ROR","asserted-by":"crossref"}]},{"name":"U.S. Department of Health & Human Services | NIH | National Institute of Biomedical Imaging and Bioengineering","id":[{"id":"https:\/\/ror.org\/00372qc85","id-type":"ROR","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Sci Data"],"abstract":"<jats:title>Abstract<\/jats:title>\n                  <jats:p>Deep learning has shown recent success in classifying anomalies in chest x-rays, but datasets are still small compared to natural image datasets. Supervision of abnormality localization has been shown to improve trained models, partially compensating for dataset sizes. However, explicitly labeling these anomalies requires an expert and is very time-consuming. We propose a potentially scalable method for collecting implicit localization data using an eye tracker to capture gaze locations and a microphone to capture a dictation of a report, imitating the setup of a reading room. The resulting REFLACX (Reports and Eye-Tracking Data for Localization of Abnormalities in Chest X-rays) dataset was labeled across five radiologists and contains 3,032 synchronized sets of eye-tracking data and timestamped report transcriptions for 2,616 chest x-rays from the MIMIC-CXR dataset. We also provide auxiliary annotations, including bounding boxes around lungs and heart and validation labels consisting of ellipses localizing abnormalities and image-level labels. Furthermore, a small subset of the data contains readings from all radiologists, allowing for the calculation of inter-rater scores.<\/jats:p>","DOI":"10.1038\/s41597-022-01441-z","type":"journal-article","created":{"date-parts":[[2022,6,18]],"date-time":"2022-06-18T09:02:33Z","timestamp":1655542953000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":54,"title":["REFLACX, a dataset of reports and eye-tracking data for localization of abnormalities in chest x-rays"],"prefix":"10.1038","volume":"9","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8740-5796","authenticated-orcid":false,"given":"Ricardo","family":"Bigolin Lanfredi","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9837-4187","authenticated-orcid":false,"given":"Mingyuan","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"William F.","family":"Auffermann","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9104-118X","authenticated-orcid":false,"given":"Jessica","family":"Chan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Phuong-Anh T.","family":"Duong","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0419-6568","authenticated-orcid":false,"given":"Vivek","family":"Srikumar","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Trafton","family":"Drew","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7451-4886","authenticated-orcid":false,"given":"Joyce D.","family":"Schroeder","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6574-0366","authenticated-orcid":false,"given":"Tolga","family":"Tasdizen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2022,6,18]]},"reference":[{"key":"1441_CR1","unstructured":"Rajpurkar, P. et al. CheXNet: Radiologist-level pneumonia detection on chest x-rays with deep learning. Preprint at http:\/\/arxiv.org\/abs\/1711.05225 (2017)."},{"key":"1441_CR2","doi-asserted-by":"publisher","first-page":"574","DOI":"10.1148\/radiol.2017162326","volume":"284","author":"P Lakhani","year":"2017","unstructured":"Lakhani, P. & Sundaram, B. Deep learning at chest radiography: Automated classification of pulmonary tuberculosis by using convolutional neural networks. Radiology 284, 574\u2013582, https:\/\/doi.org\/10.1148\/radiol.2017162326 (2017).","journal-title":"Radiology"},{"key":"1441_CR3","doi-asserted-by":"publisher","first-page":"101797","DOI":"10.1016\/j.media.2020.101797","volume":"66","author":"A Bustos","year":"2020","unstructured":"Bustos, A., Pertusa, A., Salinas, J. M. & de la Iglesia-Vay\u00e1, M. Padchest: A large chest x-ray image dataset with multi-label annotated reports. Medical Image Anal. 66, 101797, https:\/\/doi.org\/10.1016\/j.media.2020.101797 (2020).","journal-title":"Medical Image Anal."},{"key":"1441_CR4","doi-asserted-by":"publisher","unstructured":"Irvin, J. et al. CheXpert: A large chest radiograph dataset with uncertainty labels and expert comparison. In The Thirty-Third AAAI Conference on Artificial Intelligence, AAAI 2019, The Thirty-First Innovative Applications of Artificial Intelligence Conference, IAAI 2019, The Ninth AAAI Symposium on Educational Advances in Artificial Intelligence, EAAI 2019, Honolulu, Hawaii, USA, January 27 - February 1, 2019, 590\u2013597, https:\/\/doi.org\/10.1609\/aaai.v33i01.3301590 (AAAI Press, 2019).","DOI":"10.1609\/aaai.v33i01.3301590"},{"key":"1441_CR5","doi-asserted-by":"publisher","unstructured":"Wang, X. et al. ChestX-Ray8: Hospital-scale chest x-ray database and benchmarks on weakly-supervised classification and localization of common thorax diseases. In 2017 IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2017, Honolulu, HI, USA, July 21\u201326, 2017, 3462\u20133471, https:\/\/doi.org\/10.1109\/CVPR.2017.369 (IEEE Computer Society, 2017).","DOI":"10.1109\/CVPR.2017.369"},{"key":"1441_CR6","doi-asserted-by":"publisher","DOI":"10.1038\/s41597-019-0322-0","volume":"6","author":"AEW Johnson","year":"2019","unstructured":"Johnson, A. E. W. et al. MIMIC-CXR, a de-identified publicly available database of chest radiographs with free-text reports. Scientific Data 6, 317, https:\/\/doi.org\/10.1038\/s41597-019-0322-0 (2019).","journal-title":"Scientific Data"},{"key":"1441_CR7","doi-asserted-by":"publisher","unstructured":"Deng, J. et al. Imagenet: A large-scale hierarchical image database. In 2009 IEEE conference on computer vision and pattern recognition, 248\u2013255, https:\/\/doi.org\/10.1109\/CVPR.2009.5206848 (IEEE, 2009).","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"1441_CR8","doi-asserted-by":"publisher","unstructured":"Wu, B. et al. Tencent ML-Images: A large-scale multi-label image database for visual representation learning. IEEE Access 7, https:\/\/doi.org\/10.1109\/ACCESS.2019.2956775 (2019).","DOI":"10.1109\/ACCESS.2019.2956775"},{"key":"1441_CR9","doi-asserted-by":"publisher","unstructured":"Li, Z. et al. Thoracic disease identification and localization with limited supervision. In 2018 IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2018, Salt Lake City, UT, USA, June 18\u201322, 2018, 8290\u20138299, https:\/\/doi.org\/10.1109\/CVPR.2018.00865 (IEEE Computer Society, 2018).","DOI":"10.1109\/CVPR.2018.00865"},{"key":"1441_CR10","doi-asserted-by":"publisher","first-page":"e180041","DOI":"10.1148\/ryai.2019180041","volume":"1","author":"G Shih","year":"2019","unstructured":"Shih, G. et al. Augmenting the national institutes of health chest radiograph dataset with expert annotations of possible pneumonia. Radiology: Artificial Intelligence 1, e180041, https:\/\/doi.org\/10.1148\/ryai.2019180041 (2019).","journal-title":"Radiology: Artificial Intelligence"},{"key":"1441_CR11","unstructured":"Nguyen, H. Q. et al. Vindr-cxr: An open dataset of chest x-rays with radiologist\u2019s annotations. Preprint at https:\/\/arxiv.org\/abs\/2012.15029 (2021)."},{"key":"1441_CR12","doi-asserted-by":"publisher","first-page":"781","DOI":"10.1148\/radiol.2019190613","volume":"291","author":"CP Langlotz","year":"2019","unstructured":"Langlotz, C. P. et al. A roadmap for foundational research on artificial intelligence in medical imaging: From the 2018 NIH\/RSNA\/ACR\/The Academy Workshop. Radiology 291, 781\u2013791, https:\/\/doi.org\/10.1148\/radiol.2019190613 (2019).","journal-title":"Radiology"},{"key":"1441_CR13","doi-asserted-by":"publisher","unstructured":"Templier, T., Bektas, K. & Hahnloser, R. H. R. Eye-trace: Segmentation of volumetric microscopy images with eyegaze. In Kaye, J., Druin, A., Lampe, C., Morris, D. & Hourcade, J. P. (eds.) Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems, San Jose, CA, USA, May 7\u201312, 2016, 5812\u20135823, https:\/\/doi.org\/10.1145\/2858036.2858578 (ACM, 2016).","DOI":"10.1145\/2858036.2858578"},{"key":"1441_CR14","doi-asserted-by":"publisher","first-page":"597","DOI":"10.1007\/s10278-019-00220-4","volume":"32","author":"JN Stember","year":"2019","unstructured":"Stember, J. N. et al. Eye tracking for deep learning segmentation using convolutional neural networks. J. Digit. Imaging 32, 597\u2013604, https:\/\/doi.org\/10.1007\/s10278-019-00220-4 (2019).","journal-title":"J. Digit. Imaging"},{"key":"1441_CR15","doi-asserted-by":"publisher","first-page":"101","DOI":"10.1016\/j.media.2018.10.010","volume":"51","author":"N Khosravan","year":"2019","unstructured":"Khosravan, N. et al. A collaborative computer aided diagnosis (C-CAD) system with eye-tracking, sparse attentional model, and deep learning. Medical Image Anal. 51, 101\u2013115, https:\/\/doi.org\/10.1016\/j.media.2018.10.010 (2019).","journal-title":"Medical Image Anal."},{"key":"1441_CR16","doi-asserted-by":"publisher","first-page":"345","DOI":"10.1016\/j.patcog.2018.07.022","volume":"84","author":"B Gecer","year":"2018","unstructured":"Gecer, B. et al. Detection and classification of cancer in whole slide breast histopathology images using deep convolutional networks. Pattern Recognit. 84, 345\u2013356, https:\/\/doi.org\/10.1016\/j.patcog.2018.07.022 (2018).","journal-title":"Pattern Recognit."},{"key":"1441_CR17","doi-asserted-by":"publisher","first-page":"e200047","DOI":"10.1148\/ryai.2020200047","volume":"3","author":"JN Stember","year":"2021","unstructured":"Stember, J. N. et al. Integrating eye tracking and speech recognition accurately annotates mr brain images for deep learning: Proof of principle. Radiology: Artificial Intelligence 3, e200047, https:\/\/doi.org\/10.1148\/ryai.2020200047 (2021).","journal-title":"Radiology: Artificial Intelligence"},{"key":"1441_CR18","doi-asserted-by":"publisher","unstructured":"Karargyris, A. et al. Creation and validation of a chest x-ray dataset with eye-tracking and report dictation for AI development. Scientific Data 8, https:\/\/doi.org\/10.1038\/s41597-021-00863-5 (2021).","DOI":"10.1038\/s41597-021-00863-5"},{"key":"1441_CR19","doi-asserted-by":"publisher","unstructured":"Saab, K. et al. Observational supervision for medical image classification using gaze data. In Medical Image Computing and Computer Assisted Intervention - MICCAI 2021, 603\u2013614, https:\/\/doi.org\/10.1007\/978-3-030-87196-3_56 (Springer International Publishing, 2021).","DOI":"10.1007\/978-3-030-87196-3_56"},{"key":"1441_CR20","doi-asserted-by":"publisher","first-page":"697","DOI":"10.1148\/radiol.2462070712","volume":"246","author":"DM Hansell","year":"2008","unstructured":"Hansell, D. M. et al. Fleischner society: Glossary of terms for thoracic imaging. Radiology 246, 697\u2013722, https:\/\/doi.org\/10.1148\/radiol.2462070712 (2008).","journal-title":"Radiology"},{"key":"1441_CR21","doi-asserted-by":"publisher","DOI":"10.13026\/C2JT1Q","author":"AEW Johnson","year":"2019","unstructured":"Johnson, A. E. W., Pollard, T., Mark, R., Berkowitz, S. & Horng, S. The MIMIC-CXR database (version 2.0.0). PhysioNet https:\/\/doi.org\/10.13026\/C2JT1Q (2019)."},{"key":"1441_CR22","doi-asserted-by":"publisher","first-page":"e215","DOI":"10.1161\/01.CIR.101.23.e215","volume":"101","author":"AL Goldberger","year":"2000","unstructured":"Goldberger, A. L. et al. PhysioBank, PhysioToolkit, and PhysioNet: Components of a new research resource for complex physiologic signals. Circulation 101, e215\u2013e220, https:\/\/doi.org\/10.1161\/01.CIR.101.23.e215 (2000).","journal-title":"Circulation"},{"key":"1441_CR23","doi-asserted-by":"publisher","DOI":"10.13026\/8360-t248","author":"A Johnson","year":"2019","unstructured":"Johnson, A. et al. MIMIC-CXR-JPG - chest radiographs with structured labels (version 2.0.0). PhysioNet https:\/\/doi.org\/10.13026\/8360-t248 (2019)."},{"key":"1441_CR24","doi-asserted-by":"crossref","unstructured":"Johnson, A. E. W. et al. MIMIC-CXR-JPG, a large publicly available database of labeled chest radiographs. Preprint at https:\/\/arxiv.org\/abs\/1901.07042 (2019).","DOI":"10.1038\/s41597-019-0322-0"},{"key":"1441_CR25","doi-asserted-by":"publisher","first-page":"433","DOI":"10.1163\/156856897X00357","volume":"10","author":"DH Brainard","year":"1997","unstructured":"Brainard, D. H. The Psychophysics Toolbox. Spatial Vision 10, 433\u2013436, https:\/\/doi.org\/10.1163\/156856897X00357 (1997).","journal-title":"Spatial Vision"},{"key":"1441_CR26","doi-asserted-by":"publisher","first-page":"437","DOI":"10.1163\/156856897x00366","volume":"10","author":"D Pelli","year":"1997","unstructured":"Pelli, D. The VideoToolbox software for visual psychophysics: transforming numbers into movies. Spatial vision 10, 437\u2013442, https:\/\/doi.org\/10.1163\/156856897x00366 (1997).","journal-title":"Spatial vision"},{"key":"1441_CR27","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1177\/03010066070360S101","volume":"36","author":"M Kleiner","year":"2007","unstructured":"Kleiner, M. et al. What\u2019s new in psychtoolbox-3. Perception 36, 1\u201316, https:\/\/doi.org\/10.1177\/03010066070360S101 (2007).","journal-title":"Perception"},{"key":"1441_CR28","doi-asserted-by":"publisher","unstructured":"Bigolin Lanfredi, R. ricbl\/eyetracking: Code for REFLACX dataset v1.2, https:\/\/doi.org\/10.5281\/zenodo.6419833 (2022).","DOI":"10.5281\/zenodo.6419833"},{"key":"1441_CR29","doi-asserted-by":"publisher","first-page":"2","DOI":"10.2214\/ajr.15.15895","volume":"207","author":"DM Panicek","year":"2016","unstructured":"Panicek, D. M. & Hricak, H. How sure are you, doctor? a standardized lexicon to describe the radiologist\u2019s level of certainty. AJR. American journal of roentgenology 207, 2\u20133, https:\/\/doi.org\/10.2214\/ajr.15.15895 (2016).","journal-title":"AJR. American journal of roentgenology"},{"key":"1441_CR30","doi-asserted-by":"publisher","DOI":"10.13026\/S6N6-XD98","author":"A Johnson","year":"2021","unstructured":"Johnson, A. et al. MIMIC-IV (version 1.0). PhysioNet https:\/\/doi.org\/10.13026\/S6N6-XD98 (2021)."},{"key":"1441_CR31","doi-asserted-by":"publisher","DOI":"10.13026\/E0DJ-8498","author":"R Bigolin Lanfredi","year":"2021","unstructured":"Bigolin Lanfredi, R. et al. REFLACX: Reports and eye-tracking data for localization of abnormalities in chest x-rays. PhysioNet https:\/\/doi.org\/10.13026\/E0DJ-8498 (2021)."},{"key":"1441_CR32","doi-asserted-by":"publisher","first-page":"378","DOI":"10.1037\/h0031619","volume":"76","author":"J Fleiss","year":"1971","unstructured":"Fleiss, J. Measuring nominal scale agreement among many raters. Psychological Bulletin 76, 378\u2013382, https:\/\/doi.org\/10.1037\/h0031619 (1971).","journal-title":"Psychological Bulletin"},{"key":"1441_CR33","doi-asserted-by":"publisher","unstructured":"Seabold, S. & Perktold, J. statsmodels: Econometric and statistical modeling with python. In Walt, S. v. d. & Millman, J. (eds.) Proceedings of the 9th Python in Science Conference, 92\u201396, https:\/\/doi.org\/10.25080\/Majora-92bf1922-011 (2010).","DOI":"10.25080\/Majora-92bf1922-011"},{"key":"1441_CR34","doi-asserted-by":"publisher","first-page":"379","DOI":"10.1136\/bmj.331.7513.379","volume":"331","author":"Y Balabanova","year":"2005","unstructured":"Balabanova, Y. et al. Variability in interpretation of chest radiographs among russian clinicians and implications for screening programmes: observational study. BMJ 331, 379\u2013382, https:\/\/doi.org\/10.1136\/bmj.331.7513.379 (2005).","journal-title":"BMJ"},{"key":"1441_CR35","doi-asserted-by":"publisher","first-page":"111","DOI":"10.1016\/S0720-048X(01)00301-1","volume":"39","author":"LG Quekel","year":"2001","unstructured":"Quekel, L. G., Kessels, A. G., Goei, R. & van Engelshoven, J. M. Detection of lung cancer on the chest radiograph: a study on observer performance. European Journal of Radiology 39, 111\u2013116, https:\/\/doi.org\/10.1016\/S0720-048X(01)00301-1 (2001).","journal-title":"European Journal of Radiology"},{"key":"1441_CR36","doi-asserted-by":"publisher","unstructured":"Wongpakaran, N., Wongpakaran, T., Wedding, D. & Gwet, K. L. A comparison of cohen\u2019s kappa and gwet\u2019s AC1 when calculating inter-rater reliability coefficients: a study conducted with personality disorder samples. BMC Medical Research Methodology 13, https:\/\/doi.org\/10.1186\/1471-2288-13-61 (2013).","DOI":"10.1186\/1471-2288-13-61"},{"key":"1441_CR37","doi-asserted-by":"publisher","first-page":"257","DOI":"10.1093\/ptj\/85.3.257","volume":"85","author":"J Sim","year":"2005","unstructured":"Sim, J. & Wright, C. C. The kappa statistic in reliability studies: Use, interpretation, and sample size requirements. Physical Therapy 85, 257\u2013268, https:\/\/doi.org\/10.1093\/ptj\/85.3.257 (2005).","journal-title":"Physical Therapy"},{"key":"1441_CR38","doi-asserted-by":"publisher","unstructured":"Le Meur, O. & Baccino, T. Methods for comparing scanpaths and saliency maps: strengths and weaknesses. Behavior Research Methods 1\u201316, https:\/\/doi.org\/10.3758\/s13428-012-0226-9 (2012).","DOI":"10.3758\/s13428-012-0226-9"},{"key":"1441_CR39","doi-asserted-by":"publisher","unstructured":"Bruny\u00e9, T. T. et al. Pupil diameter changes reflect difficulty and diagnostic accuracy during medical image interpretation. BMC Medical Informatics and Decision Making 16, https:\/\/doi.org\/10.1186\/s12911-016-0322-3 (2016).","DOI":"10.1186\/s12911-016-0322-3"},{"key":"1441_CR40","doi-asserted-by":"publisher","unstructured":"McFee, B. et al. librosa\/librosa: 0.8.0, https:\/\/doi.org\/10.5281\/zenodo.3955228 (2020).","DOI":"10.5281\/zenodo.3955228"},{"key":"1441_CR41","doi-asserted-by":"publisher","first-page":"357","DOI":"10.1038\/s41586-020-2649-2","volume":"585","author":"CR Harris","year":"2020","unstructured":"Harris, C. R. et al. Array programming with NumPy. Nature 585, 357\u2013362, https:\/\/doi.org\/10.1038\/s41586-020-2649-2 (2020).","journal-title":"Nature"},{"key":"1441_CR42","doi-asserted-by":"publisher","unstructured":"McKinney, W. Data Structures for Statistical Computing in Python. In S., van der Walt & J. Millman (eds.) Proceedings of the 9th Python in Science Conference, 56\u201361, https:\/\/doi.org\/10.25080\/Majora-92bf1922-00a (2010).","DOI":"10.25080\/Majora-92bf1922-00a"},{"key":"1441_CR43","doi-asserted-by":"publisher","unstructured":"Reback, J. et al. pandas-dev\/pandas: Pandas 1.1.1, https:\/\/doi.org\/10.5281\/zenodo.3993412 (2020).","DOI":"10.5281\/zenodo.3993412"},{"key":"1441_CR44","doi-asserted-by":"publisher","first-page":"90","DOI":"10.1109\/MCSE.2007.55","volume":"9","author":"JD Hunter","year":"2007","unstructured":"Hunter, J. D. Matplotlib: A 2d graphics environment. Computing in Science Engineering 9, 90\u201395, https:\/\/doi.org\/10.1109\/MCSE.2007.55 (2007).","journal-title":"Computing in Science Engineering"},{"key":"1441_CR45","doi-asserted-by":"publisher","unstructured":"Caswell, T. A. et al. matplotlib\/matplotlib: Rel: v3.5.1, https:\/\/doi.org\/10.5281\/zenodo.5773480 (2021).","DOI":"10.5281\/zenodo.5773480"},{"key":"1441_CR46","unstructured":"Gillies, S. et al. Shapely: manipulation and analysis of geometric objects. GitHub https:\/\/github.com\/Toblerity\/Shapely (2007)."},{"key":"1441_CR47","doi-asserted-by":"publisher","unstructured":"van der Walt, S. et al. scikit-image: image processing in Python. PeerJ 2, e453, https:\/\/doi.org\/10.7717\/peerj.453 (2014).","DOI":"10.7717\/peerj.453"},{"key":"1441_CR48","doi-asserted-by":"publisher","unstructured":"Mason, D. et al. pydicom\/pydicom: pydicom 2.1.2, https:\/\/doi.org\/10.5281\/zenodo.4313150 (2020).","DOI":"10.5281\/zenodo.4313150"},{"key":"1441_CR49","doi-asserted-by":"publisher","unstructured":"van Kemenade, H. et al. python-pillow\/pillow 8.0.1, https:\/\/doi.org\/10.5281\/zenodo.4118627 (2020).","DOI":"10.5281\/zenodo.4118627"},{"key":"1441_CR50","first-page":"2825","volume":"12","author":"F Pedregosa","year":"2011","unstructured":"Pedregosa, F. et al. Scikit-learn: Machine learning in Python. Journal of Machine Learning Research 12, 2825\u20132830 (2011).","journal-title":"Journal of Machine Learning Research"},{"key":"1441_CR51","unstructured":"Bird, S., Klein, E. & Loper, E. Natural language processing with Python: analyzing text with the natural language toolkit (\u201cO\u2019Reilly Media, Inc.\u201d, 2009)."},{"key":"1441_CR52","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.4781125","author":"Zulko","year":"2021","unstructured":"Zulko et al. johncooper199\/moviepy. Zenodo https:\/\/doi.org\/10.5281\/zenodo.4781125 (2021)."},{"key":"1441_CR53","unstructured":"Bradski, G. The OpenCV Library. Dr. Dobb\u2019s Journal of Software Tools (2000)."}],"container-title":["Scientific Data"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41597-022-01441-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41597-022-01441-z","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41597-022-01441-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,11,24]],"date-time":"2022-11-24T14:31:33Z","timestamp":1669300293000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41597-022-01441-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,18]]},"references-count":53,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2022,12]]}},"alternative-id":["1441"],"URL":"https:\/\/doi.org\/10.1038\/s41597-022-01441-z","relation":{"references":[{"id-type":"doi","id":"10.13026\/C2JT1Q","asserted-by":"subject"},{"id-type":"doi","id":"10.13026\/8360-t248","asserted-by":"subject"},{"id-type":"doi","id":"10.13026\/S6N6-XD98","asserted-by":"subject"},{"id-type":"doi","id":"10.13026\/E0DJ-8498","asserted-by":"subject"},{"id-type":"doi","id":"10.5281\/zenodo.4781125","asserted-by":"subject"}]},"ISSN":["2052-4463"],"issn-type":[{"value":"2052-4463","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,6,18]]},"assertion":[{"value":"8 November 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"1 June 2022","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"18 June 2022","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"350"}}