{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,4]],"date-time":"2026-07-04T16:38:00Z","timestamp":1783183080855,"version":"3.54.6"},"reference-count":12,"publisher":"Oxford University Press (OUP)","issue":"18","license":[{"start":{"date-parts":[[2019,2,6]],"date-time":"2019-02-06T00:00:00Z","timestamp":1549411200000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000002","name":"U.S. National Institutes of Health","doi-asserted-by":"crossref","id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"crossref"}]},{"name":"National Cancer Institute grants","award":["U01CA220401"],"award-info":[{"award-number":["U01CA220401"]}]},{"name":"National Cancer Institute grants","award":["U24CA194362"],"award-info":[{"award-number":["U24CA194362"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2019,9,15]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:sec>\n                  <jats:title>Motivation<\/jats:title>\n                  <jats:p>While deep-learning algorithms have demonstrated outstanding performance in semantic image segmentation tasks, large annotation datasets are needed to create accurate models. Annotation of histology images is challenging due to the effort and experience required to carefully delineate tissue structures, and difficulties related to sharing and markup of whole-slide images.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Results<\/jats:title>\n                  <jats:p>We recruited 25 participants, ranging in experience from senior pathologists to medical students, to delineate tissue regions in 151 breast cancer slides using the Digital Slide Archive. Inter-participant discordance was systematically evaluated, revealing low discordance for tumor and stroma, and higher discordance for more subjectively defined or rare tissue classes. Feedback provided by senior participants enabled the generation and curation of 20\u00a0000+ annotated tissue regions. Fully convolutional networks trained using these annotations were highly accurate (mean AUC=0.945), and the scale of annotation data provided notable improvements in image classification accuracy.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Availability and Implementation<\/jats:title>\n                  <jats:p>Dataset is freely available at: https:\/\/goo.gl\/cNM4EL.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Supplementary information<\/jats:title>\n                  <jats:p>Supplementary data are available at Bioinformatics online.<\/jats:p>\n               <\/jats:sec>","DOI":"10.1093\/bioinformatics\/btz083","type":"journal-article","created":{"date-parts":[[2019,2,5]],"date-time":"2019-02-05T12:33:02Z","timestamp":1549369982000},"page":"3461-3467","source":"Crossref","is-referenced-by-count":295,"title":["Structured crowdsourcing enables convolutional segmentation of histology images"],"prefix":"10.1093","volume":"35","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7599-6162","authenticated-orcid":false,"given":"Mohamed","family":"Amgad","sequence":"first","affiliation":[{"name":"Department of Biomedical Informatics, Emory University School of Medicine , Atlanta, GA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Habiba","family":"Elfandy","sequence":"additional","affiliation":[{"name":"Department of Pathology, National Cancer Institute , Cairo, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hagar","family":"Hussein","sequence":"additional","affiliation":[{"name":"Department of Medicine, Cairo University , Cairo, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lamees A","family":"Atteya","sequence":"additional","affiliation":[{"name":"Egyptian Ministry of Health , Cairo, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mai A T","family":"Elsebaie","sequence":"additional","affiliation":[{"name":"Department of Medicine, Ain Shams University , Cairo, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lamia S","family":"Abo Elnasr","sequence":"additional","affiliation":[{"name":"Department of Medicine, Menoufia University , Menoufia, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rokia A","family":"Sakr","sequence":"additional","affiliation":[{"name":"Department of Medicine, Menoufia University , Menoufia, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hazem S E","family":"Salem","sequence":"additional","affiliation":[{"name":"Department of Medicine, Ain Shams University , Cairo, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ahmed F","family":"Ismail","sequence":"additional","affiliation":[{"name":"Department of Pathology, Medical Research Institute, Alexandria University , Alexandria, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Anas M","family":"Saad","sequence":"additional","affiliation":[{"name":"Department of Medicine, Ain Shams University , Cairo, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Joumana","family":"Ahmed","sequence":"additional","affiliation":[{"name":"Department of Medicine, Cairo University , Cairo, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Maha A T","family":"Elsebaie","sequence":"additional","affiliation":[{"name":"Department of Medicine, Ain Shams University , Cairo, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mustafijur","family":"Rahman","sequence":"additional","affiliation":[{"name":"Department of Medicine, Chittagong University , Chittagong, Bangladesh"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Inas A","family":"Ruhban","sequence":"additional","affiliation":[{"name":"Department of Medicine, Damascus University , Damascus, Syria"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nada M","family":"Elgazar","sequence":"additional","affiliation":[{"name":"Department of Medicine, Mansoura University , Mansoura, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yahya","family":"Alagha","sequence":"additional","affiliation":[{"name":"Department of Medicine, Cairo University , Cairo, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mohamed H","family":"Osman","sequence":"additional","affiliation":[{"name":"Department of Medicine, Zagazig University , Zagazig, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ahmed M","family":"Alhusseiny","sequence":"additional","affiliation":[{"name":"Department of Medicine, Mansoura University , Mansoura, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mariam M","family":"Khalaf","sequence":"additional","affiliation":[{"name":"Department of Medicine, Batterjee Medical College , Jeddah, Saudi Arabia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Abo-Alela F","family":"Younes","sequence":"additional","affiliation":[{"name":"Department of Medicine, Ain Shams University , Cairo, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ali","family":"Abdulkarim","sequence":"additional","affiliation":[{"name":"Department of Medicine, Cairo University , Cairo, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Duaa M","family":"Younes","sequence":"additional","affiliation":[{"name":"Department of Medicine, Ain Shams University , Cairo, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ahmed M","family":"Gadallah","sequence":"additional","affiliation":[{"name":"Department of Medicine, Ain Shams University , Cairo, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ahmad M","family":"Elkashash","sequence":"additional","affiliation":[{"name":"Department of Medicine, Cairo University , Cairo, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Salma Y","family":"Fala","sequence":"additional","affiliation":[{"name":"Department of Medicine, Suez Canal University , Ismailia, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Basma M","family":"Zaki","sequence":"additional","affiliation":[{"name":"Department of Medicine, Suez Canal University , Ismailia, Egypt"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jonathan","family":"Beezley","sequence":"additional","affiliation":[{"name":"Kitware Inc., Clifton Park , NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Deepak R","family":"Chittajallu","sequence":"additional","affiliation":[{"name":"Kitware Inc., Clifton Park , NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"David","family":"Manthey","sequence":"additional","affiliation":[{"name":"Kitware Inc., Clifton Park , NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"David A","family":"Gutman","sequence":"additional","affiliation":[{"name":"Department of Neurology, Emory University School of Medicine , Atlanta, GA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3504-4965","authenticated-orcid":false,"given":"Lee A D","family":"Cooper","sequence":"additional","affiliation":[{"name":"Department of Biomedical Informatics, Emory University School of Medicine , Atlanta, GA, USA"},{"name":"Department of Biomedical Engineering, Emory University , Atlanta, GA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"286","published-online":{"date-parts":[[2019,2,6]]},"reference":[{"key":"2023013108054510300_btz083-B1","doi-asserted-by":"crossref","first-page":"2.","DOI":"10.4103\/jpi.jpi_65_17","article-title":"A review on the applications of crowdsourcing in human pathology","volume":"9","author":"Alialy","year":"2018","journal-title":"J. Pathol. Inform"},{"key":"2023013108054510300_btz083-B3","first-page":"1016","article-title":"Revisiting the hallmarks of cancer","volume":"7","author":"Fouad","year":"2017","journal-title":"Am. J. Cancer Res"},{"key":"2023013108054510300_btz083-B4","doi-asserted-by":"crossref","first-page":"1091","DOI":"10.1136\/amiajnl-2012-001469","article-title":"Cancer Digital Slide Archive: an informatics resource to support integrated in silico analysis of TCGA pathology data","volume":"20","author":"Gutman","year":"2013","journal-title":"J. Am. Med. Inform. Assoc"},{"key":"2023013108054510300_btz083-B5","doi-asserted-by":"crossref","first-page":"e75","DOI":"10.1158\/0008-5472.CAN-17-0629","article-title":"The digital slide archive: a software platform for management, integration, and analysis of histology for cancer research","volume":"77","author":"Gutman","year":"2017","journal-title":"Cancer Res"},{"key":"2023013108054510300_btz083-B6","doi-asserted-by":"crossref","first-page":"587","DOI":"10.1038\/s41592-018-0069-0","article-title":"Quanti.us: a tool for rapid, flexible, crowd-based annotation of images","volume":"15","author":"Hughes","year":"2018","journal-title":"Nat. Methods"},{"key":"2023013108054510300_btz083-B7","first-page":"294","article-title":"Crowdsourcing image annotation for nucleus detection and segmentation in computational pathology: evaluating experts, automated methods, and the crowd","author":"Irshad","year":"2015","journal-title":"Pac. Symp. Biocomput"},{"key":"2023013108054510300_btz083-B8","doi-asserted-by":"crossref","first-page":"43286.","DOI":"10.1038\/srep43286","article-title":"Crowdsourcing scoring of immunohistochemistry images: evaluating Performance of the Crowd and an Automated Computational Method","volume":"7","author":"Irshad","year":"2017","journal-title":"Sci. Rep"},{"key":"2023013108054510300_btz083-B9","doi-asserted-by":"crossref","first-page":"177","DOI":"10.1561\/0600000071","article-title":"Crowdsourcing in computer vision","volume":"10","author":"Kovashka","year":"2016","journal-title":"Found. Trends Comput. Graph. Vis"},{"key":"2023013108054510300_btz083-B10","first-page":"3431","author":"Long","year":"2015"},{"key":"2023013108054510300_btz083-B11","doi-asserted-by":"crossref","first-page":"e4614.","DOI":"10.1097\/MD.0000000000004614","article-title":"Features of triple-negative breast cancer: analysis of 38, 813 cases from the national cancer database","volume":"95","author":"Plasilova","year":"2016","journal-title":"Medicine"},{"key":"2023013108054510300_btz083-B12","doi-asserted-by":"crossref","first-page":"34","DOI":"10.1109\/38.946629","article-title":"Color transfer between images","volume":"21","author":"Reinhard","year":"2001","journal-title":"IEEE Comput. Graph"},{"key":"2023013108054510300_btz083-B14","volume-title":"Workshops at the Twenty-Sixth AAAI Conference on Artificial Intelligence","author":"Su","year":"2012"}],"container-title":["Bioinformatics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article-pdf\/35\/18\/3461\/48975491\/bioinformatics_35_18_3461.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article-pdf\/35\/18\/3461\/48975491\/bioinformatics_35_18_3461.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,31]],"date-time":"2023-01-31T13:45:46Z","timestamp":1675172746000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article\/35\/18\/3461\/5307750"}},"subtitle":[],"editor":[{"given":"Robert","family":"Murphy","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"editor"}]}],"short-title":[],"issued":{"date-parts":[[2019,2,6]]},"references-count":12,"journal-issue":{"issue":"18","published-print":{"date-parts":[[2019,9,15]]}},"URL":"https:\/\/doi.org\/10.1093\/bioinformatics\/btz083","relation":{},"ISSN":["1367-4803","1367-4811"],"issn-type":[{"value":"1367-4803","type":"print"},{"value":"1367-4811","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2019,9,15]]},"published":{"date-parts":[[2019,2,6]]}}}