{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"institution":[{"name":"bioRxiv"}],"indexed":{"date-parts":[[2026,10,18]],"date-time":"2026-10-18T02:19:13Z","timestamp":1792289953484,"version":"4.4.0"},"posted":{"date-parts":[[2026,8,6]]},"group-title":"Bioinformatics","reference-count":62,"publisher":"openRxiv","license":[{"start":{"date-parts":[[2026,8,6]],"date-time":"2026-08-06T00:00:00Z","timestamp":1785974400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.biorxiv.org\/about\/FAQ#license"}],"funder":[{"name":"Eric and Wendy Schmidt Center"},{"name":"NIDDK\/NIH","award":["5RC2DK135492-02"],"award-info":[{"award-number":["5RC2DK135492-02"]}]},{"name":"NCCIH\/NIH","award":["1DP2AT012345"],"award-info":[{"award-number":["1DP2AT012345"]}]},{"name":"ONR","award":["N00014-24-1-2687"],"award-info":[{"award-number":["N00014-24-1-2687"]}]},{"name":"DOE","award":["DE-SC0023187"],"award-info":[{"award-number":["DE-SC0023187"]}]},{"name":"MIT J-Clinic for Machine Learning and Health"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"accepted":{"date-parts":[[2026,8,6]]},"abstract":"<jats:title>Abstract<\/jats:title>\n                <jats:p>Paired histopathology and spatial transcriptomics data are advancing our understanding of tissue biology and disease, but modeling both modalities at single-cell resolution while mapping local and distal cell-cell interdependencies remains computationally prohibitive. Here we introduce TissueFormer, a framework for pretraining foundation models with linear rather than quadratic computational complexity, overcoming a long-standing barrier to modeling long-range dependencies at scale. Trained on over 17 million image-expression pairs from 1.2K tissue slides, TissueFormer excels at predicting spatial gene expression from histology images at cellular resolution and scales to diagnostic tasks at the cell, region, and slide levels. Additionally, by identifying both long and short-range cell-cell interdependencies, our model enables the generation of testable hypotheses about disease mechanisms and staging, as demonstrated in lung fibrosis and breast cancer.<\/jats:p>","DOI":"10.64898\/2026.07.31.741265","type":"posted-content","created":{"date-parts":[[2026,8,7]],"date-time":"2026-08-07T04:40:19Z","timestamp":1786077619000},"source":"Crossref","is-referenced-by-count":2,"title":["Multi-Modal Foundation Model with Whole-Slide Attention Enables Transferrable Digital Pathology at Single-Cell Resolution"],"prefix":"10.64898","author":[{"given":"Qitian","family":"Wu","sequence":"first","affiliation":[{"name":"Eric and Wendy Schmidt Center, Broad Institute of MIT and Harvard, Cambridge, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Qiyu","family":"Gong","sequence":"additional","affiliation":[{"name":"Broad Institute of MIT and Harvard, Cambridge, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Luezhen","family":"Yuan","sequence":"additional","affiliation":[{"name":"Eric and Wendy Schmidt Center, Broad Institute of MIT and Harvard, Cambridge, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhiyi","family":"Li","sequence":"additional","affiliation":[{"name":"Department of Electrical Engineering and Computer Science, Massachusetts Institute of Technology, Cambridge, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Orr","family":"Ashenberg","sequence":"additional","affiliation":[{"name":"Eric and Wendy Schmidt Center, Broad Institute of MIT and Harvard, Cambridge, MA, USA"},{"name":"Klarman Cell Observatory, Broad Institute of MIT and Harvard, Cambridge, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2308-3649","authenticated-orcid":false,"given":"Fei","family":"Chen","sequence":"additional","affiliation":[{"name":"Broad Institute of MIT and Harvard, Cambridge, MA, USA"},{"name":"Department of Stem Cell and Regenerative Biology, Harvard University, Cambridge, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ramnik","family":"Xavier","sequence":"additional","affiliation":[{"name":"Broad Institute of MIT and Harvard, Cambridge, MA, USA"},{"name":"Klarman Cell Observatory, Broad Institute of MIT and Harvard, Cambridge, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Caroline","family":"Uhler","sequence":"additional","affiliation":[{"name":"Eric and Wendy Schmidt Center, Broad Institute of MIT and Harvard, Cambridge, MA, USA"},{"name":"Department of Electrical Engineering and Computer Science, Massachusetts Institute of Technology, Cambridge, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"54368","reference":[{"key":"2026081003251333000_2026.07.31.741265v1.1","doi-asserted-by":"publisher","DOI":"10.1126\/science.aaw1219"},{"key":"2026081003251333000_2026.07.31.741265v1.2","doi-asserted-by":"publisher","DOI":"10.1038\/s41587-020-0739-1"},{"key":"2026081003251333000_2026.07.31.741265v1.3","doi-asserted-by":"publisher","DOI":"10.1038\/S41587-022-01483-Z"},{"issue":"5","key":"2026081003251333000_2026.07.31.741265v1.4","doi-asserted-by":"crossref","first-page":"534","DOI":"10.1038\/s41592-022-01409-2","article-title":"Museum of spatial transcriptomics","volume":"19","year":"2022","journal-title":"Nature Methods"},{"key":"2026081003251333000_2026.07.31.741265v1.5","doi-asserted-by":"publisher","DOI":"10.1038\/s41588-022-01268-9"},{"issue":"7","key":"2026081003251333000_2026.07.31.741265v1.6","doi-asserted-by":"crossref","first-page":"872","DOI":"10.1038\/s41551-024-01205-7","article-title":"Spatial multi-omics at subcellular resolution via high-throughput in situ pairwise sequencing","volume":"8","year":"2024","journal-title":"Nature Biomedical Engineering"},{"issue":"1","key":"2026081003251333000_2026.07.31.741265v1.7","doi-asserted-by":"crossref","first-page":"31","DOI":"10.1038\/s41467-020-20249-2","article-title":"Multi-domain translation between single-cell imaging and sequencing data using autoencoders","volume":"12","year":"2021","journal-title":"Nature communications"},{"issue":"1","key":"2026081003251333000_2026.07.31.741265v1.8","doi-asserted-by":"crossref","first-page":"7480","DOI":"10.1038\/s41467-022-35233-1","article-title":"Graph-based autoencoder integrates spatial transcriptomics with chromatin images and identifies joint biomarkers for alzheimer\u2019s disease","volume":"13","year":"2022","journal-title":"Nature Communications"},{"key":"2026081003251333000_2026.07.31.741265v1.9","doi-asserted-by":"publisher","DOI":"10.1016\/j.ccell.2022.07.004"},{"key":"2026081003251333000_2026.07.31.741265v1.10","doi-asserted-by":"publisher","DOI":"10.1038\/nature21065"},{"key":"2026081003251333000_2026.07.31.741265v1.11","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-022-05060-x"},{"issue":"1","key":"2026081003251333000_2026.07.31.741265v1.12","doi-asserted-by":"crossref","first-page":"6112","DOI":"10.1038\/s41467-024-50285-1","article-title":"Unsupervised representation learning of chromatin images identifies changes in cell state and tissue organization in dcis","volume":"15","year":"2024","journal-title":"Nature Communications"},{"key":"2026081003251333000_2026.07.31.741265v1.13","doi-asserted-by":"publisher","DOI":"10.1038\/s41591-019-0508-1"},{"key":"2026081003251333000_2026.07.31.741265v1.14","doi-asserted-by":"publisher","DOI":"10.1158\/0008-5472.CAN-22-2794"},{"issue":"32","key":"2026081003251333000_2026.07.31.741265v1.15","doi-asserted-by":"crossref","first-page":"5473","DOI":"10.1126\/sciadv.adl5473","article-title":"Spatial transcriptomic characterization of pathologic niches in ipf","volume":"10","year":"2024","journal-title":"Science Advances"},{"key":"2026081003251333000_2026.07.31.741265v1.16","doi-asserted-by":"publisher","DOI":"10.1126\/science.aaf2403"},{"key":"2026081003251333000_2026.07.31.741265v1.17","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-020-2363-0"},{"issue":"1","key":"2026081003251333000_2026.07.31.741265v1.18","doi-asserted-by":"crossref","first-page":"3877","DOI":"10.1038\/s41467-020-17678-4","article-title":"A deep learning model to predict rna-seq expression of tumours from whole slide images","volume":"11","year":"2020","journal-title":"Nature communications"},{"issue":"5","key":"2026081003251333000_2026.07.31.741265v1.19","first-page":"668","article-title":"Pathology-based transcriptomics identifies tumor microenvironment characteristics in colorectal cancer","volume":"54","year":"2022","journal-title":"Nature Genetics"},{"key":"2026081003251333000_2026.07.31.741265v1.20","unstructured":"Brown, T. , Mann, B. , Ryder, N. , Subbiah, M. , Kaplan, J.D. , Dhariwal, P. , Neelakantan, A. , Shyam, P. , Sastry, G. , Askell, A. , et al.: Language models are few-shot learners. In: Advances in Neural Information Processing Systems, pp. 1877\u20131901 (2020)"},{"key":"2026081003251333000_2026.07.31.741265v1.21","unstructured":"Bommasani, R. , Hudson, D.A. , Adeli, E. , Altman, R. , Arora, S. , Arx, S. , Bernstein, M.S. , Bohg, J. , Bosselut, A. , Brunskill, E. , et al.: On the opportunities and risks of foundation models. CoRR abs\/2108.07258 (2021)"},{"key":"2026081003251333000_2026.07.31.741265v1.22","doi-asserted-by":"publisher","DOI":"10.1038\/s41591-023-02448-8"},{"key":"2026081003251333000_2026.07.31.741265v1.23","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-025-08710-y"},{"key":"2026081003251333000_2026.07.31.741265v1.24","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-024-07441-w"},{"key":"2026081003251333000_2026.07.31.741265v1.25","doi-asserted-by":"publisher","DOI":"10.1038\/s41591-024-02857-3"},{"key":"2026081003251333000_2026.07.31.741265v1.26","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-024-07894-z"},{"key":"2026081003251333000_2026.07.31.741265v1.27","doi-asserted-by":"publisher","DOI":"10.1038\/s41591-024-03141-0"},{"key":"2026081003251333000_2026.07.31.741265v1.28","doi-asserted-by":"crossref","unstructured":"Fu, X. , Cao, Y. , Bian, B. , Wang, C. , Graham, D. , Pathmanathan, N. , Patrick, E. , Kim, J. , Yang, J.Y.H. : Spatial gene expression at single-cell resolution from histology using deep learning with ghist. Nature Methods, 1\u201311 (2025)","DOI":"10.1101\/2024.07.02.601790"},{"key":"2026081003251333000_2026.07.31.741265v1.29","unstructured":"Chen, W. , Zhang, P. , Tran, T.N. , Xiao, Y. , Li, S. , Shah, V.V. , Cheng, H. , Brannan, K.W. , Youker, K. , Lai, L. , et al.: A visual\u2013omics foundation model to bridge histopathology with spatial transcriptomics. Nature Methods, 1\u201315 (2025)"},{"key":"2026081003251333000_2026.07.31.741265v1.30","doi-asserted-by":"publisher","DOI":"10.1016\/j.devcel.2017.11.021"},{"key":"2026081003251333000_2026.07.31.741265v1.31","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-023-05793-3"},{"issue":"1","key":"2026081003251333000_2026.07.31.741265v1.32","doi-asserted-by":"crossref","first-page":"196","DOI":"10.1038\/s41392-024-01888-z","article-title":"Cell\u2013cell communication: new insights and clinical implications","volume":"9","year":"2024","journal-title":"Signal Transduction and Targeted Therapy"},{"issue":"1","key":"2026081003251333000_2026.07.31.741265v1.33","first-page":"001","article-title":"Applications of transformer-based language models in bioinformatics: a survey","volume":"3","year":"2023","journal-title":"Bioinformatics Advances"},{"key":"2026081003251333000_2026.07.31.741265v1.34","doi-asserted-by":"crossref","unstructured":"Jaume, G. , Doucet, P. , Song, A. , Lu, M.Y. , Almagro P\u00e9rez, C. , Wagner, S. , Vaidya, A. , Chen, R. , Williamson, D. , Kim, A. , et al.: Hest-1k: A dataset for spatial transcriptomics and histology image analysis. In: Advances in Neural Information Processing Systems, pp. 53798\u201353833 (2024)","DOI":"10.52202\/079017-1704"},{"key":"2026081003251333000_2026.07.31.741265v1.35","unstructured":"Bioptimus: H-optimus-1. https:\/\/huggingface.co\/bioptimus\/H-optimus-1"},{"key":"2026081003251333000_2026.07.31.741265v1.36","doi-asserted-by":"crossref","unstructured":"Vannan, A. , Lyu, R. , Williams, A.L. , Negretti, N.M. , Mee, E.D. , Hirsh, J. , Hirsh, S. , Hadad, N. , Nichols, D.S. , Calvi, C.L. , et al.: Spatial transcriptomics identifies molecular niche dysregulation associated with distal lung remodeling in pulmonary fibrosis. Nature Genetics, 1\u201312 (2025)","DOI":"10.1038\/s41588-025-02080-x"},{"issue":"1","key":"2026081003251333000_2026.07.31.741265v1.37","doi-asserted-by":"crossref","first-page":"8353","DOI":"10.1038\/s41467-023-43458-x","article-title":"High resolution mapping of the tumor microenvironment using integrated single-cell, spatial and in situ analysis","volume":"14","year":"2023","journal-title":"Nature communications"},{"key":"2026081003251333000_2026.07.31.741265v1.38","unstructured":"Vaswani, A. , Shazeer, N. , Parmar, N. , Uszkoreit, J. , Jones, L. , Gomez, A.N. , Kaiser, L . , Polosukhin, I. : Attention is all you need. Advances in neural information processing systems 30 (2017)"},{"key":"2026081003251333000_2026.07.31.741265v1.39","unstructured":"Chen, T. , Kornblith, S. , Norouzi, M. , Hinton, G. : A simple framework for contrastive learning of visual representations. In: International Conference on Machine Learning, pp. 1597\u20131607 (2020)"},{"key":"2026081003251333000_2026.07.31.741265v1.40","unstructured":"Radford, A. , Kim, J.W. , Hallacy, C. , Ramesh, A. , Goh, G. , Agarwal, S. , Sastry, G. , Askell, A. , Mishkin, P. , Clark, J. , et al.: Learning transferable visual models from natural language supervision. In: International Conference on Machine Learning, pp. 8748\u20138763 (2021)"},{"key":"2026081003251333000_2026.07.31.741265v1.41","doi-asserted-by":"publisher","DOI":"10.1038\/s41379-021-00919-2"},{"key":"2026081003251333000_2026.07.31.741265v1.42","doi-asserted-by":"publisher","DOI":"10.3892\/ijo.2023.5526"},{"key":"2026081003251333000_2026.07.31.741265v1.43","doi-asserted-by":"crossref","unstructured":"Liang, S. , Tang, J. , Wang, G. , Ma, J. : Steamboat: Attention-based multiscale delineation of cellular interactions in tissues. In: International Conference on Research in Computational Molecular Biology, pp. 432\u2013436 (2025). Springer","DOI":"10.1007\/978-3-031-90252-9_55"},{"key":"2026081003251333000_2026.07.31.741265v1.44","doi-asserted-by":"crossref","unstructured":"Akbarnejad, A. , Steele, L. , Jafree, D.J. , Birk, S. , Sallese, M.R. , Rademaker, K. , Boxall, A. , Rumney, B. , Tudor, C. , Patel, M. , et al.: Mapping and reprogramming human tissue microenvironments with mintflow. bioRxiv, 2025\u201306 (2025)","DOI":"10.1101\/2025.06.24.661094"},{"key":"2026081003251333000_2026.07.31.741265v1.45","doi-asserted-by":"crossref","unstructured":"Schroeder, A. , Loth, M.L. , Luo, C. , Yao, S. , Yan, H. , Zhang, D. , Piya, S. , Plowey, E. , Hu, W. , Clemenceau, J.R. , et al.: Scaling up spatial transcriptomics for large-sized tissues: uncovering cellular-level tissue architecture beyond conventional platforms with iscale. Nature Methods, 1\u201312 (2025)","DOI":"10.1101\/2025.02.25.640190"},{"issue":"4","key":"2026081003251333000_2026.07.31.741265v1.46","doi-asserted-by":"crossref","first-page":"89","DOI":"10.3892\/ijmm.2024.5413","article-title":"Spp1 promotes the polarization of m2 macrophages through the jak2\/stat3 signaling pathway and accelerates the progression of idiopathic pulmonary fibrosis","volume":"54","year":"2024","journal-title":"International Journal of Molecular Medicine"},{"key":"2026081003251333000_2026.07.31.741265v1.47","doi-asserted-by":"crossref","unstructured":"Hoeft, K. , Schaefer, G.J. , Kim, H. , Schumacher, D. , Bleckwehl, T. , Long, Q. , Klinkhammer, B.M. , Peisker, F. , Koch, L. , Nagai, J. , et al.: Platelet-instructed spp1+ macrophages drive myofibroblast activation in fibrosis in a cxcl4-dependent manner. Cell reports 42(2) (2023)","DOI":"10.1016\/j.celrep.2023.112131"},{"issue":"5","key":"2026081003251333000_2026.07.31.741265v1.48","doi-asserted-by":"crossref","first-page":"345","DOI":"10.3390\/cells14050345","article-title":"Pro-fibrotic macrophage subtypes: Spp1+ macrophages as a key player and therapeutic target in cardiac fibrosis?","volume":"14","year":"2025","journal-title":"Cells"},{"issue":"1","key":"2026081003251333000_2026.07.31.741265v1.49","doi-asserted-by":"crossref","first-page":"21","DOI":"10.1038\/s41467-017-02481-5","article-title":"Macrophages orchestrate breast cancer early dissemination and metastasis","volume":"9","year":"2018","journal-title":"Nature communications"},{"key":"2026081003251333000_2026.07.31.741265v1.50","doi-asserted-by":"publisher","DOI":"10.1038\/cr.2017.34"},{"key":"2026081003251333000_2026.07.31.741265v1.51","doi-asserted-by":"publisher","DOI":"10.1016\/j.cell.2021.12.023"},{"key":"2026081003251333000_2026.07.31.741265v1.52","unstructured":"MGI Data and Statistical Reports. https:\/\/www.informatics.jax.org\/downloads\/reports\/index.tml#homology"},{"key":"2026081003251333000_2026.07.31.741265v1.53","doi-asserted-by":"publisher","DOI":"10.1016\/j.cell.2019.05.031"},{"key":"2026081003251333000_2026.07.31.741265v1.54","unstructured":"Kipf, T.N. , Welling, M. : Semi-supervised classification with graph convolutional networks. In: International Conference on Learning Representations (2017)"},{"key":"2026081003251333000_2026.07.31.741265v1.55","article-title":"Representation learning with contrastive predictive coding","year":"2018","journal-title":"arXiv preprint"},{"key":"2026081003251333000_2026.07.31.741265v1.56","unstructured":"Zhang, Y. , Jiang, H. , Miura, Y. , Manning, C.D. , Langlotz, C.P. : Contrastive learning of medical visual representations from paired images and text. In: Machine Learning for Healthcare Conference, pp. 2\u201325 (2022)"},{"key":"2026081003251333000_2026.07.31.741265v1.57","unstructured":"Radford, A. , Kim, J.W. , Hallacy, C. , Ramesh, A. , Goh, G. , Agarwal, S. , Sastry, G. , Askell, A. , Mishkin, P. , Clark, J. , et al.: Learning transferable visual models from natural language supervision. In: International Conference on Machine Learning, pp. 8748\u20138763 (2021)"},{"key":"2026081003251333000_2026.07.31.741265v1.58","unstructured":"Daunhawer, I. , Bizeul, A. , Palumbo, E. , Marx, A. , Vogt, J.E. : Identifiability results for multimodal contrastive learning. In: International Conference on Learning Representations (2023)"},{"key":"2026081003251333000_2026.07.31.741265v1.59","unstructured":"Wang, T. , Isola, P. : Understanding contrastive representation learning through alignment and uniformity on the hypersphere. In: International Conference on Machine Learning, pp. 9929\u20139939 (2020)"},{"key":"2026081003251333000_2026.07.31.741265v1.60","unstructured":"Von Oswald, J. , Niklasson, E. , Randazzo, E. , Sacramento, J. , Mordvintsev, A. , Zhmoginov, A. , Vladymyrov, M. : Transformers learn in-context by gradient descent. In: International Conference on Machine Learning, pp. 35151\u201335174 (2023)"},{"key":"2026081003251333000_2026.07.31.741265v1.61","unstructured":"OpenAI: Gpt-4 technical report (2024). https:\/\/arxiv.org\/abs\/2303.08774"},{"key":"2026081003251333000_2026.07.31.741265v1.62","doi-asserted-by":"crossref","unstructured":"Lin, T.-Y. , Goyal, P. , Girshick, R. , He, K. , Doll\u00e1r, P. : Focal loss for dense object detection. In: IEEE International Conference on Computer Vision, pp. 2980\u20132988 (2017)","DOI":"10.1109\/ICCV.2017.324"}],"container-title":[],"original-title":[],"link":[{"URL":"https:\/\/syndication.highwire.org\/content\/doi\/10.64898\/2026.07.31.741265","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,8,10]],"date-time":"2026-08-10T10:25:24Z","timestamp":1786357524000},"score":1,"resource":{"primary":{"URL":"http:\/\/biorxiv.org\/lookup\/doi\/10.64898\/2026.07.31.741265"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,8,6]]},"references-count":62,"URL":"https:\/\/doi.org\/10.64898\/2026.07.31.741265","relation":{},"subject":[],"published":{"date-parts":[[2026,8,6]]},"subtype":"preprint"}}