{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,26]],"date-time":"2026-02-26T23:59:52Z","timestamp":1772150392578,"version":"3.50.1"},"reference-count":36,"publisher":"Oxford University Press (OUP)","issue":"19","license":[{"start":{"date-parts":[[2022,8,13]],"date-time":"2022-08-13T00:00:00Z","timestamp":1660348800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/academic.oup.com\/journals\/pages\/open_access\/funder_policies\/chorus\/standard_publication_model"}],"funder":[{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["U01CA242936"],"award-info":[{"award-number":["U01CA242936"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R01EY028450"],"award-info":[{"award-number":["R01EY028450"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R01CA214928"],"award-info":[{"award-number":["R01CA214928"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R01CA239120"],"award-info":[{"award-number":["R01CA239120"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["ACI 1443054"],"award-info":[{"award-number":["ACI 1443054"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["IIS 1350885"],"award-info":[{"award-number":["IIS 1350885"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003593","name":"CNPq","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100003593","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004901","name":"FAPEMIG","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100004901","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022,9,30]]},"abstract":"<jats:title>Abstract<\/jats:title>\n                  <jats:sec>\n                    <jats:title>Motivation<\/jats:title>\n                    <jats:p>Predicting pathological complete response (pCR) to neoadjuvant chemotherapy (NAC) in triple-negative breast cancer (TNBC) patients accurately is direly needed for clinical decision making. pCR is also regarded as a strong predictor of overall survival. In this work, we propose a deep learning system to predict pCR to NAC based on serial pathology images stained with hematoxylin and eosin and two immunohistochemical biomarkers (Ki67 and PHH3). To support human prior domain knowledge-based guidance and enhance interpretability of the deep learning system, we introduce a human knowledge-derived spatial attention mechanism to inform deep learning models of informative tissue areas of interest. For each patient, three serial breast tumor tissue sections from biopsy blocks were sectioned, stained in three different stains and integrated. The resulting comprehensive attention information from the image triplets is used to guide our prediction system for prognostic tissue regions.<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Results<\/jats:title>\n                    <jats:p>The experimental dataset consists of 26\u00a0419 pathology image patches of 1000\u00d71000 pixels from 73 TNBC patients treated with NAC. Image patches from randomly selected 43 patients are used as a training dataset and images patches from the rest 30 are used as a testing dataset. By the maximum voting from patch-level results, our proposed model achieves a 93% patient-level accuracy, outperforming baselines and other state-of-the-art systems, suggesting its high potential for clinical decision making.<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Availability and implementation<\/jats:title>\n                    <jats:p>The codes, the documentation and example data are available on an open source at: https:\/\/github.com\/jkonglab\/PCR_Prediction_Serial_WSIs_biomarkers<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Supplementary information<\/jats:title>\n                    <jats:p>Supplementary data are available at Bioinformatics online.<\/jats:p>\n                  <\/jats:sec>","DOI":"10.1093\/bioinformatics\/btac558","type":"journal-article","created":{"date-parts":[[2022,8,13]],"date-time":"2022-08-13T14:07:57Z","timestamp":1660399677000},"page":"4605-4612","source":"Crossref","is-referenced-by-count":36,"title":["A spatial attention guided deep learning system for prediction of pathological complete response using breast cancer histopathology images"],"prefix":"10.1093","volume":"38","author":[{"given":"Hongyi","family":"Duanmu","sequence":"first","affiliation":[{"name":"Department of Computer Science, Stony Brook University , Stony Brook, NY, USA"}]},{"given":"Shristi","family":"Bhattarai","sequence":"additional","affiliation":[{"name":"Department of Biology, Georgia State University , Atlanta, GA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6114-8215","authenticated-orcid":false,"given":"Hongxiao","family":"Li","sequence":"additional","affiliation":[{"name":"Department of Mathematics and Statistics and Computer Science, Georgia State University , Atlanta, GA, USA"}]},{"given":"Zhan","family":"Shi","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Stony Brook University , Stony Brook, NY, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9369-9361","authenticated-orcid":false,"given":"Fusheng","family":"Wang","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Stony Brook University , Stony Brook, NY, USA"},{"name":"Department of Biomedical Informatics, Stony Brook University , Stony Brook, NY, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6289-3914","authenticated-orcid":false,"given":"George","family":"Teodoro","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Federal University of Minas Gerais , Belo Horizonte, MG, Brazil"}]},{"given":"Keerthi","family":"Gogineni","sequence":"additional","affiliation":[{"name":"Department of Hematology-Medical Oncology, Winship Cancer Institute, Emory University School of Medicine , Atlanta, GA, USA"},{"name":"Department of Surgery, Winship Cancer Institute, Emory University School of Medicine , Atlanta, GA, USA"},{"name":"Georgia Cancer Center for Excellence, Grady Health System , Atlanta, GA, USA"}]},{"given":"Preeti","family":"Subhedar","sequence":"additional","affiliation":[{"name":"Georgia Cancer Center for Excellence, Grady Health System , Atlanta, GA, USA"}]},{"given":"Umay","family":"Kiraz","sequence":"additional","affiliation":[{"name":"Department of Pathology, Stavanger University Hospital , Stavanger, Norway"}]},{"given":"Emiel A M","family":"Janssen","sequence":"additional","affiliation":[{"name":"Department of Pathology, Stavanger University Hospital , Stavanger, Norway"},{"name":"Department of Chemistry, Bioscience and Environmental Engineering, University of Stavanger , Stavanger, Norway"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4489-5320","authenticated-orcid":false,"given":"Ritu","family":"Aneja","sequence":"additional","affiliation":[{"name":"Department of Clinical and Diagnostic Sciences, School of Health Professions, University of Alabama at Birmingham , Birmingham, AL, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8757-8339","authenticated-orcid":false,"given":"Jun","family":"Kong","sequence":"additional","affiliation":[{"name":"Department of Mathematics and Statistics and Computer Science, Georgia State University , Atlanta, GA, USA"},{"name":"Winship Cancer Institute, Emory University , Atlanta, GA, USA"}]}],"member":"286","published-online":{"date-parts":[[2022,8,13]]},"reference":[{"key":"2023081301104593600_btac558-B1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/s13058-016-0682-8","article-title":"Computational pathology of pre-treatment biopsies identifies lymphocyte density as a predictor of response to neoadjuvant chemotherapy in breast cancer","volume":"18","author":"Ali","year":"2016","journal-title":"Breast Cancer Res"},{"key":"2023081301104593600_btac558-B2","doi-asserted-by":"crossref","first-page":"1832","DOI":"10.1093\/annonc\/mdx266","article-title":"Lymphocyte density determined by computational pathology validated as a predictor of response to neoadjuvant chemotherapy in breast cancer: secondary analysis of the Artemis trial","volume":"28","author":"Ali","year":"2017","journal-title":"Ann. Oncol"},{"key":"2023081301104593600_btac558-B3","doi-asserted-by":"crossref","first-page":"95","DOI":"10.1038\/s42256-019-0019-2","article-title":"Deep-learning cardiac motion analysis for human survival prediction","volume":"1","author":"Bello","year":"2019","journal-title":"Nat. Mach. Intell"},{"key":"2023081301104593600_btac558-B5","doi-asserted-by":"crossref","first-page":"164","DOI":"10.1016\/S0140-6736(13)62422-8","article-title":"Pathological complete response and long-term clinical benefit in breast cancer: the CTNeoBC pooled analysis","volume":"384","author":"Cortazar","year":"2014","journal-title":"Lancet"},{"key":"2023081301104593600_btac558-B6","doi-asserted-by":"crossref","first-page":"346","DOI":"10.1038\/nature10983","article-title":"The genomic and transcriptomic architecture of 2,000 breast tumours reveals novel subgroups","volume":"486","author":"Curtis","year":"2012","journal-title":"Nature"},{"key":"2023081301104593600_btac558-B7","doi-asserted-by":"crossref","first-page":"95","DOI":"10.1007\/978-1-4419-8339-8_13","volume-title":"Selected Works of Murray Rosenblatt, New York","author":"Davis","year":"2011"},{"key":"2023081301104593600_btac558-B8","doi-asserted-by":"crossref","first-page":"1504","DOI":"10.1038\/sj.bjc.6603756","article-title":"Ki67 as prognostic marker in early breast cancer: a meta-analysis of published studies involving 12,155 patients","volume":"96","author":"De Azambuja","year":"2007","journal-title":"Br. J. Cancer"},{"key":"2023081301104593600_btac558-B9","doi-asserted-by":"crossref","first-page":"1776","DOI":"10.1002\/cncr.27995","article-title":"Magnetic resonance imaging as a predictor of pathologic response in patients treated with neoadjuvant systemic treatment for operable breast cancer: translational breast cancer research consortium trial 017","volume":"119","author":"De Los Santos","year":"2013","journal-title":"Cancer"},{"key":"2023081301104593600_btac558-B10","doi-asserted-by":"crossref","first-page":"379","DOI":"10.1007\/s10549-020-06093-4","article-title":"Analysis of tumor nuclear features using artificial intelligence to predict response to neoadjuvant chemotherapy in high-risk breast cancer patients","volume":"186","author":"Dodington","year":"2021","journal-title":"Breast Cancer Res. Treat"},{"key":"2023081301104593600_btac558-B11","first-page":"242","volume-title":"International Conference on Medical Image Computing and Computer-Assisted Intervention, Lima, Peru","author":"Duanmu","year":"2020"},{"key":"2023081301104593600_btac558-B12","doi-asserted-by":"crossref","first-page":"3905","DOI":"10.1093\/bioinformatics\/btab418","article-title":"Foveal blur-boosted segmentation of nuclei in histopathology images with shape prior knowledge and probability map constraints","volume":"37","author":"Duanmu","year":"2021","journal-title":"Bioinformatics"},{"key":"2023081301104593600_btac558-B13","first-page":"550","volume-title":"International Conference on Medical Image Computing and Computer-Assisted Intervention, Strasbourg, France","author":"Duanmu","year":"2021"},{"key":"2023081301104593600_btac558-B14","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1080\/21681163.2015.1124249","article-title":"Holistic classification of CT attenuation patterns for interstitial lung diseases via deep convolutional neural networks","volume":"6","author":"Gao","year":"2018","journal-title":"Comput. Methods Biomech. Biomed. Eng. Imaging Vis"},{"key":"2023081301104593600_btac558-B15","doi-asserted-by":"crossref","first-page":"629","DOI":"10.1097\/PAT.0000000000000320","article-title":"Prognostic relevance of proliferation markers (Ki67, PHH3) within the cross-relation of ERG translocation and androgen receptor expression in prostate cancer","volume":"47","author":"Goltz","year":"2015","journal-title":"Pathology"},{"key":"2023081301104593600_btac558-B16","first-page":"770","author":"He","year":"2016"},{"key":"2023081301104593600_btac558-B17","first-page":"2961","author":"He","year":"2017"},{"key":"2023081301104593600_btac558-B18","doi-asserted-by":"crossref","first-page":"436","DOI":"10.1038\/nature14539","article-title":"Deep learning","volume":"521","author":"LeCun","year":"2015","journal-title":"Nature"},{"key":"2023081301104593600_btac558-B19","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/s12967-021-03020-z","article-title":"Deep learning-based predictive biomarker of pathological complete response to neoadjuvant chemotherapy from histological images in breast cancer","volume":"19","author":"Li","year":"2021","journal-title":"J. Transl. Med"},{"key":"2023081301104593600_btac558-B20","doi-asserted-by":"crossref","first-page":"1116","DOI":"10.1007\/s00259-020-04684-3","article-title":"18 F-FDG PET\/CT radiomic predictors of pathologic complete response (pCR) to neoadjuvant chemotherapy in breast cancer patients","volume":"47","author":"Li","year":"2020","journal-title":"Eur. J. Nucl. Med. Mol. Imaging"},{"key":"2023081301104593600_btac558-B21","doi-asserted-by":"crossref","first-page":"1275","DOI":"10.1200\/JCO.2007.14.4147","article-title":"Response to neoadjuvant therapy and long-term survival in patients with triple-negative breast cancer","volume":"26","author":"Liedtke","year":"2008","journal-title":"J. Clin. Oncol"},{"key":"2023081301104593600_btac558-B22","doi-asserted-by":"crossref","first-page":"e115103","DOI":"10.1371\/journal.pone.0115103","article-title":"The value of tumor infiltrating lymphocytes (TILs) for predicting response to neoadjuvant chemotherapy in breast cancer: a systematic review and meta-analysis","volume":"9","author":"Mao","year":"2014","journal-title":"PLoS One"},{"key":"2023081301104593600_btac558-B23","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1155\/2020\/3189691","article-title":"An evaluation of deep learning methods for small object detection","volume":"2020","author":"Nguyen","year":"2020","journal-title":"J. Electr. Comput. Eng"},{"key":"2023081301104593600_btac558-B24","first-page":"4421","article-title":"MELK expression in breast cancer is associated with infiltration of immune cell and pathological compete response (pCR) after neoadjuvant chemotherapy","volume":"11","author":"Oshi","year":"2021","journal-title":"Am. J. Cancer Res"},{"key":"2023081301104593600_btac558-B25","doi-asserted-by":"crossref","first-page":"18","DOI":"10.1007\/s00330-011-2236-x","article-title":"Comparison of diffusion-weighted MR imaging and FDG PET\/CT to predict pathological complete response to neoadjuvant chemotherapy in patients with breast cancer","volume":"22","author":"Park","year":"2012","journal-title":"Eur. Radiol"},{"key":"2023081301104593600_btac558-B26","doi-asserted-by":"crossref","first-page":"231","DOI":"10.1016\/j.ecolmodel.2005.03.026","article-title":"Maximum entropy modeling of species geographic distributions","volume":"190","author":"Phillips","year":"2006","journal-title":"Ecol. Modell"},{"key":"2023081301104593600_btac558-B27","first-page":"7077","author":"Prakash","year":"2021"},{"key":"2023081301104593600_btac558-B28","first-page":"424","author":"Rossetti"},{"key":"2023081301104593600_btac558-B29","first-page":"291","article-title":"Quantification of histochemical staining by color deconvolution","volume":"23","author":"Ruifrok","year":"2001","journal-title":"Anal. Quant. Cytol. Histol"},{"key":"2023081301104593600_btac558-B30","doi-asserted-by":"crossref","first-page":"221","DOI":"10.1146\/annurev-bioeng-071516-044442","article-title":"Deep learning in medical image analysis","volume":"19","author":"Shen","year":"2017","journal-title":"Annu. Rev. Biomed. Eng"},{"key":"2023081301104593600_btac558-B31","first-page":"192","author":"Shin","year":"2021"},{"key":"2023081301104593600_btac558-B32","doi-asserted-by":"crossref","first-page":"1796","DOI":"10.1200\/JCO.2011.38.8595","article-title":"Definition and impact of pathologic complete response on prognosis after neoadjuvant chemotherapy in various intrinsic breast cancer subtypes","volume":"30","author":"Von Minckwitz","year":"2012","journal-title":"J. Clin. Oncol"},{"key":"2023081301104593600_btac558-B33","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1038\/s41598-021-85353-9","article-title":"Quantitative analysis of contrast enhanced spectral mammography grey value for early prediction of pathological response of breast cancer to neoadjuvant chemotherapy","volume":"11","author":"Xing","year":"2021","journal-title":"Sci. Rep"},{"key":"2023081301104593600_btac558-B34","doi-asserted-by":"crossref","first-page":"101789","DOI":"10.1016\/j.media.2020.101789","article-title":"Whole slide images based cancer survival prediction using attention guided deep multiple instance learning networks","volume":"65","author":"Yao","year":"2020","journal-title":"Med. Image Anal"},{"key":"2023081301104593600_btac558-B35","first-page":"2921","article-title":"Learning deep features for discriminative localization","author":"Zhou","year":"2016"},{"key":"2023081301104593600_btac558-B36","doi-asserted-by":"crossref","first-page":"100004","DOI":"10.1016\/j.array.2019.100004","article-title":"A review: deep learning for medical image segmentation using multi-modality fusion","volume":"3\u20134","author":"Zhou","year":"2019","journal-title":"Array"},{"key":"2023081301104593600_btac558-B37","first-page":"6688","author":"Zhu","year":"2019"}],"container-title":["Bioinformatics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/bioinformatics\/advance-article-pdf\/doi\/10.1093\/bioinformatics\/btac558\/45500712\/btac558.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article-pdf\/38\/19\/4605\/49885491\/btac558.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article-pdf\/38\/19\/4605\/49885491\/btac558.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,8,12]],"date-time":"2023-08-12T21:12:01Z","timestamp":1691874721000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article\/38\/19\/4605\/6665902"}},"subtitle":[],"editor":[{"given":"Hanchuan","family":"Peng","sequence":"additional","affiliation":[]}],"short-title":[],"issued":{"date-parts":[[2022,8,13]]},"references-count":36,"journal-issue":{"issue":"19","published-print":{"date-parts":[[2022,9,30]]}},"URL":"https:\/\/doi.org\/10.1093\/bioinformatics\/btac558","relation":{"has-preprint":[{"id-type":"doi","id":"10.1101\/2022.05.25.493468","asserted-by":"object"}]},"ISSN":["1367-4803","1367-4811"],"issn-type":[{"value":"1367-4803","type":"print"},{"value":"1367-4811","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2022,10,1]]},"published":{"date-parts":[[2022,8,13]]}}}