{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,13]],"date-time":"2026-03-13T08:48:56Z","timestamp":1773391736398,"version":"3.50.1"},"update-to":[{"DOI":"10.1371\/journal.pcbi.1009439","type":"new_version","label":"New version","source":"publisher","updated":{"date-parts":[[2021,10,4]],"date-time":"2021-10-04T00:00:00Z","timestamp":1633305600000}}],"reference-count":100,"publisher":"Public Library of Science (PLoS)","issue":"9","license":[{"start":{"date-parts":[[2021,9,22]],"date-time":"2021-09-22T00:00:00Z","timestamp":1632268800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100000324","name":"Gatsby Charitable Foundation","doi-asserted-by":"crossref","award":["GAT3708"],"award-info":[{"award-number":["GAT3708"]}],"id":[{"id":"10.13039\/501100000324","id-type":"DOI","asserted-by":"crossref"}]},{"name":"McKnight Foundaton"},{"DOI":"10.13039\/100005237","name":"Helen Hay Whitney Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100005237","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100013368","name":"Deutsche Akademie der Naturforscher Leopoldina - Nationale Akademie der Wissenschaften","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100013368","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001675","name":"International Brain Research Organization","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100001675","id-type":"DOI","asserted-by":"publisher"}]},{"name":"National Science Foundation","award":["DBI-1707398"],"award-info":[{"award-number":["DBI-1707398"]}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R21MH116348"],"award-info":[{"award-number":["R21MH116348"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"crossref","award":["RF1MH120680"],"award-info":[{"award-number":["RF1MH120680"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["T32NS064929"],"award-info":[{"award-number":["T32NS064929"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["T32MH015144"],"award-info":[{"award-number":["T32MH015144"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["U19NS107613"],"award-info":[{"award-number":["U19NS107613"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["U19NS107613"],"award-info":[{"award-number":["U19NS107613"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["U19NS107613"],"award-info":[{"award-number":["U19NS107613"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["U19NS107613"],"award-info":[{"award-number":["U19NS107613"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["U19NS107613"],"award-info":[{"award-number":["U19NS107613"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["U19NS107613"],"award-info":[{"award-number":["U19NS107613"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["UF1NS107696"],"award-info":[{"award-number":["UF1NS107696"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000893","name":"Simons Foundation","doi-asserted-by":"publisher","award":["542963"],"award-info":[{"award-number":["542963"]}],"id":[{"id":"10.13039\/100000893","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000893","name":"Simons Foundation","doi-asserted-by":"publisher","award":["542963"],"award-info":[{"award-number":["542963"]}],"id":[{"id":"10.13039\/100000893","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000893","name":"Simons Foundation","doi-asserted-by":"publisher","award":["542963"],"award-info":[{"award-number":["542963"]}],"id":[{"id":"10.13039\/100000893","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000893","name":"Simons Foundation","doi-asserted-by":"publisher","award":["543023"],"award-info":[{"award-number":["543023"]}],"id":[{"id":"10.13039\/100000893","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000893","name":"Simons Foundation","doi-asserted-by":"publisher","award":["543023"],"award-info":[{"award-number":["543023"]}],"id":[{"id":"10.13039\/100000893","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000893","name":"Simons Foundation","doi-asserted-by":"publisher","award":["543023"],"award-info":[{"award-number":["543023"]}],"id":[{"id":"10.13039\/100000893","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000893","name":"Simons Foundation","doi-asserted-by":"publisher","award":["543023"],"award-info":[{"award-number":["543023"]}],"id":[{"id":"10.13039\/100000893","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000893","name":"Simons Foundation","doi-asserted-by":"publisher","award":["543023"],"award-info":[{"award-number":["543023"]}],"id":[{"id":"10.13039\/100000893","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000893","name":"Simons Foundation","doi-asserted-by":"publisher","award":["543023"],"award-info":[{"award-number":["543023"]}],"id":[{"id":"10.13039\/100000893","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000893","name":"Simons Foundation","doi-asserted-by":"publisher","award":["543023"],"award-info":[{"award-number":["543023"]}],"id":[{"id":"10.13039\/100000893","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100010269","name":"Wellcome Trust","doi-asserted-by":"publisher","award":["209558"],"award-info":[{"award-number":["209558"]}],"id":[{"id":"10.13039\/100010269","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100010269","name":"Wellcome Trust","doi-asserted-by":"publisher","award":["209558"],"award-info":[{"award-number":["209558"]}],"id":[{"id":"10.13039\/100010269","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100010269","name":"Wellcome Trust","doi-asserted-by":"publisher","award":["209558"],"award-info":[{"award-number":["209558"]}],"id":[{"id":"10.13039\/100010269","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100004440","name":"Wellcome Trust","doi-asserted-by":"publisher","award":["209558"],"award-info":[{"award-number":["209558"]}],"id":[{"id":"10.13039\/100004440","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100010269","name":"Wellcome Trust","doi-asserted-by":"publisher","award":["216324"],"award-info":[{"award-number":["216324"]}],"id":[{"id":"10.13039\/100010269","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100010269","name":"Wellcome Trust","doi-asserted-by":"publisher","award":["216324"],"award-info":[{"award-number":["216324"]}],"id":[{"id":"10.13039\/100010269","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100010269","name":"Wellcome Trust","doi-asserted-by":"publisher","award":["216324"],"award-info":[{"award-number":["216324"]}],"id":[{"id":"10.13039\/100010269","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100004440","name":"Wellcome Trust","doi-asserted-by":"publisher","award":["216324"],"award-info":[{"award-number":["216324"]}],"id":[{"id":"10.13039\/100004440","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["www.ploscompbiol.org"],"crossmark-restriction":false},"short-container-title":["PLoS Comput Biol"],"abstract":"<jats:p>Recent neuroscience studies demonstrate that a deeper understanding of brain function requires a deeper understanding of behavior. Detailed behavioral measurements are now often collected using video cameras, resulting in an increased need for computer vision algorithms that extract useful information from video data. Here we introduce a new video analysis tool that combines the output of supervised pose estimation algorithms (e.g. DeepLabCut) with unsupervised dimensionality reduction methods to produce interpretable, low-dimensional representations of behavioral videos that extract more information than pose estimates alone. We demonstrate this tool by extracting interpretable behavioral features from videos of three different head-fixed mouse preparations, as well as a freely moving mouse in an open field arena, and show how these interpretable features can facilitate downstream behavioral and neural analyses. We also show how the behavioral features produced by our model improve the precision and interpretation of these downstream analyses compared to using the outputs of either fully supervised or fully unsupervised methods alone.<\/jats:p>","DOI":"10.1371\/journal.pcbi.1009439","type":"journal-article","created":{"date-parts":[[2021,9,22]],"date-time":"2021-09-22T14:34:40Z","timestamp":1632321280000},"page":"e1009439","update-policy":"https:\/\/doi.org\/10.1371\/journal.pcbi.corrections_policy","source":"Crossref","is-referenced-by-count":31,"title":["Partitioning variability in animal behavioral videos using semi-supervised variational autoencoders"],"prefix":"10.1371","volume":"17","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3756-1349","authenticated-orcid":true,"given":"Matthew R.","family":"Whiteway","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2054-8439","authenticated-orcid":true,"given":"Dan","family":"Biderman","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4966-6869","authenticated-orcid":true,"given":"Yoni","family":"Friedman","sequence":"additional","affiliation":[]},{"given":"Mario","family":"Dipoppa","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1448-5662","authenticated-orcid":true,"given":"E. Kelly","family":"Buchanan","sequence":"additional","affiliation":[]},{"given":"Anqi","family":"Wu","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3565-5376","authenticated-orcid":true,"given":"John","family":"Zhou","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5228-6918","authenticated-orcid":true,"given":"Niccol\u00f2","family":"Bonacchi","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8587-4919","authenticated-orcid":true,"given":"Nathaniel J.","family":"Miska","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5297-3363","authenticated-orcid":true,"given":"Jean-Paul","family":"Noel","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5966-714X","authenticated-orcid":true,"given":"Erica","family":"Rodriguez","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4854-3034","authenticated-orcid":true,"given":"Michael","family":"Schartner","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1677-1489","authenticated-orcid":true,"given":"Karolina","family":"Socha","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5270-6513","authenticated-orcid":true,"given":"Anne E.","family":"Urai","sequence":"additional","affiliation":[]},{"given":"C. Daniel","family":"Salzman","sequence":"additional","affiliation":[]},{"name":"The International Brain Laboratory","sequence":"additional","affiliation":[]},{"given":"John P.","family":"Cunningham","sequence":"additional","affiliation":[]},{"given":"Liam","family":"Paninski","sequence":"additional","affiliation":[]}],"member":"340","published-online":{"date-parts":[[2021,9,22]]},"reference":[{"issue":"1","key":"pcbi.1009439.ref001","doi-asserted-by":"crossref","first-page":"18","DOI":"10.1016\/j.neuron.2014.09.005","article-title":"Toward a science of computational ethology","volume":"84","author":"DJ Anderson","year":"2014","journal-title":"Neuron"},{"issue":"11","key":"pcbi.1009439.ref002","doi-asserted-by":"crossref","first-page":"1455","DOI":"10.1038\/nn.3812","article-title":"Big behavioral data: psychology, ethology and the foundations of neuroscience","volume":"17","author":"A Gomez-Marin","year":"2014","journal-title":"Nature neuroscience"},{"issue":"3","key":"pcbi.1009439.ref003","doi-asserted-by":"crossref","first-page":"480","DOI":"10.1016\/j.neuron.2016.12.041","article-title":"Neuroscience needs behavior: correcting a reductionist bias","volume":"93","author":"JW Krakauer","year":"2017","journal-title":"Neuron"},{"issue":"1","key":"pcbi.1009439.ref004","doi-asserted-by":"crossref","first-page":"23","DOI":"10.1186\/s12915-018-0494-7","article-title":"Measuring behavior across scales","volume":"16","author":"GJ Berman","year":"2018","journal-title":"BMC biology"},{"issue":"1","key":"pcbi.1009439.ref005","doi-asserted-by":"crossref","first-page":"11","DOI":"10.1016\/j.neuron.2019.09.038","article-title":"Computational neuroethology: a call to action","volume":"104","author":"SR Datta","year":"2019","journal-title":"Neuron"},{"key":"pcbi.1009439.ref006","first-page":"1","article-title":"Quantifying behavior to understand the brain","author":"TD Pereira","year":"2020","journal-title":"Nature Neuroscience"},{"issue":"35","key":"pcbi.1009439.ref007","doi-asserted-by":"crossref","first-page":"7551","DOI":"10.1523\/JNEUROSCI.1920-17.2018","article-title":"Beyond trial-based paradigms: Continuous behavior, ongoing neural activity, and natural stimuli","volume":"38","author":"A Huk","year":"2018","journal-title":"Journal of Neuroscience"},{"issue":"10","key":"pcbi.1009439.ref008","doi-asserted-by":"crossref","first-page":"1632","DOI":"10.1111\/2041-210X.13256","article-title":"Applications for deep learning in ecology","volume":"10","author":"S Christin","year":"2019","journal-title":"Methods in Ecology and Evolution"},{"key":"pcbi.1009439.ref009","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/j.conb.2019.10.008","article-title":"Deep learning tools for the measurement of animal behavior in neuroscience","volume":"60","author":"MW Mathis","year":"2020","journal-title":"Current opinion in neurobiology"},{"issue":"9","key":"pcbi.1009439.ref010","doi-asserted-by":"crossref","first-page":"1281","DOI":"10.1038\/s41593-018-0209-y","article-title":"DeepLabCut: markerless pose estimation of user-defined body parts with deep learning","volume":"21","author":"A Mathis","year":"2018","journal-title":"Nature neuroscience"},{"key":"pcbi.1009439.ref011","doi-asserted-by":"crossref","first-page":"e47994","DOI":"10.7554\/eLife.47994","article-title":"DeepPoseKit, a software toolkit for fast and robust animal pose estimation using deep learning","volume":"8","author":"JM Graving","year":"2019","journal-title":"Elife"},{"issue":"1","key":"pcbi.1009439.ref012","doi-asserted-by":"crossref","first-page":"117","DOI":"10.1038\/s41592-018-0234-5","article-title":"Fast animal pose estimation using deep neural networks","volume":"16","author":"TD Pereira","year":"2019","journal-title":"Nature methods"},{"key":"pcbi.1009439.ref013","doi-asserted-by":"crossref","unstructured":"Wu A, Buchanan EK, Whiteway M, Schartner M, Meijer G, Noel JP, et al. Deep Graph Pose: a semi-supervised deep graphical model for improved animal pose tracking. In: Advances in Neural Information Processing Systems; 2020. p. 6040\u20136052.","DOI":"10.1101\/2020.08.20.259705"},{"issue":"2","key":"pcbi.1009439.ref014","doi-asserted-by":"crossref","first-page":"181","DOI":"10.1016\/j.cub.2017.12.002","article-title":"Structure of the zebrafish locomotor repertoire revealed with unsupervised behavioral clustering","volume":"28","author":"JC Marques","year":"2018","journal-title":"Current Biology"},{"key":"pcbi.1009439.ref015","article-title":"VAE-SNE: a deep generative model for simultaneous dimensionality reduction and clustering","author":"JM Graving","year":"2020","journal-title":"BioRxiv"},{"key":"pcbi.1009439.ref016","article-title":"Identifying Behavioral Structure from Deep Variational Embeddings of Animal Motion","author":"K Luxem","year":"2020","journal-title":"bioRxiv"},{"issue":"1","key":"pcbi.1009439.ref017","doi-asserted-by":"crossref","first-page":"54","DOI":"10.1016\/j.cub.2019.11.022","article-title":"Deconstructing hunting behavior reveals a tightly coupled stimulus-response loop","volume":"30","author":"DS Mearns","year":"2020","journal-title":"Current Biology"},{"issue":"6414","key":"pcbi.1009439.ref018","doi-asserted-by":"crossref","first-page":"584","DOI":"10.1126\/science.aau2013","article-title":"Efficient cortical coding of 3D posture in freely behaving rats","volume":"362","author":"B Mimica","year":"2018","journal-title":"Science"},{"issue":"4","key":"pcbi.1009439.ref019","doi-asserted-by":"crossref","first-page":"e1007791","DOI":"10.1371\/journal.pcbi.1007791","article-title":"Localized semi-nonnegative matrix factorization (LocaNMF) of widefield calcium imaging data","volume":"16","author":"S Saxena","year":"2020","journal-title":"PLOS Computational Biology"},{"issue":"4","key":"pcbi.1009439.ref020","doi-asserted-by":"crossref","first-page":"e1000028","DOI":"10.1371\/journal.pcbi.1000028","article-title":"Dimensionality and dynamics in the behavior of C. elegans","volume":"4","author":"GJ Stephens","year":"2008","journal-title":"PLoS Comput Biol"},{"issue":"99","key":"pcbi.1009439.ref021","doi-asserted-by":"crossref","first-page":"20140672","DOI":"10.1098\/rsif.2014.0672","article-title":"Mapping the stereotyped behaviour of freely moving fruit flies","volume":"11","author":"GJ Berman","year":"2014","journal-title":"Journal of The Royal Society Interface"},{"issue":"10","key":"pcbi.1009439.ref022","doi-asserted-by":"crossref","first-page":"1677","DOI":"10.1038\/s41593-019-0502-4","article-title":"Single-trial neural dynamics are dominated by richly varied movements","volume":"22","author":"S Musall","year":"2019","journal-title":"Nature neuroscience"},{"issue":"6437","key":"pcbi.1009439.ref023","doi-asserted-by":"crossref","DOI":"10.1126\/science.aav7893","article-title":"Spontaneous behaviors drive multidimensional, brainwide activity","volume":"364","author":"C Stringer","year":"2019","journal-title":"Science"},{"issue":"6","key":"pcbi.1009439.ref024","doi-asserted-by":"crossref","first-page":"1121","DOI":"10.1016\/j.neuron.2015.11.031","article-title":"Mapping sub-second structure in mouse behavior","volume":"88","author":"AB Wiltschko","year":"2015","journal-title":"Neuron"},{"issue":"1","key":"pcbi.1009439.ref025","doi-asserted-by":"crossref","first-page":"44","DOI":"10.1016\/j.cell.2018.04.019","article-title":"The striatum organizes 3D behavior via moment-to-moment action selection","volume":"174","author":"JE Markowitz","year":"2018","journal-title":"Cell"},{"key":"pcbi.1009439.ref026","unstructured":"Johnson M, Duvenaud DK, Wiltschko A, Adams RP, Datta SR. Composing graphical models with neural networks for structured representations and fast inference. In: Advances in Neural Information Processing Systems; 2016. p. 2946\u20132954."},{"key":"pcbi.1009439.ref027","unstructured":"Batty E, Whiteway M, Saxena S, Biderman D, Abe T, Musall S, et al. BehaveNet: nonlinear embedding and Bayesian neural decoding of behavioral videos. In: Advances in Neural Information Processing Systems; 2019. p. 15706\u201315717."},{"key":"pcbi.1009439.ref028","unstructured":"Kingma DP, Welling M. Auto-encoding variational bayes. arXiv preprint arXiv:13126114. 2013."},{"key":"pcbi.1009439.ref029","unstructured":"Rezende DJ, Mohamed S, Wierstra D. Stochastic backpropagation and approximate inference in deep generative models. arXiv preprint arXiv:14014082. 2014."},{"key":"pcbi.1009439.ref030","doi-asserted-by":"crossref","unstructured":"Yu S, Yu K, Tresp V, Kriegel HP, Wu M. Supervised probabilistic principal component analysis. In: Proceedings of the 12th ACM SIGKDD international conference on Knowledge discovery and data mining; 2006. p. 464\u2013473.","DOI":"10.1145\/1150402.1150454"},{"key":"pcbi.1009439.ref031","unstructured":"Zhuang F, Cheng X, Luo P, Pan SJ, He Q. Supervised representation learning: Transfer learning with deep autoencoders. In: Twenty-Fourth International Joint Conference on Artificial Intelligence; 2015."},{"key":"pcbi.1009439.ref032","doi-asserted-by":"crossref","unstructured":"Gogna A, Majumdar A. Semi supervised autoencoder. In: International Conference on Neural Information Processing. Springer; 2016. p. 82\u201389.","DOI":"10.1007\/978-3-319-46672-9_10"},{"key":"pcbi.1009439.ref033","unstructured":"Pu Y, Gan Z, Henao R, Yuan X, Li C, Stevens A, et al. Variational autoencoder for deep learning of images, labels and captions. In: Advances in Neural Information Processing Systems; 2016. p. 2352\u20132360."},{"key":"pcbi.1009439.ref034","doi-asserted-by":"crossref","first-page":"42","DOI":"10.1016\/j.neucom.2015.03.110","article-title":"Deep extreme learning machines: supervised autoencoding architecture for classification","volume":"174","author":"MD Tissera","year":"2016","journal-title":"Neurocomputing"},{"key":"pcbi.1009439.ref035","unstructured":"Le L, Patterson A, White M. Supervised autoencoders: Improving generalization performance with unsupervised regularizers. In: Advances in Neural Information Processing Systems; 2018. p. 107\u2013117."},{"key":"pcbi.1009439.ref036","unstructured":"Miller A, Obermeyer Z, Cunningham J, Mullainathan S. Discriminative Regularization for Latent Variable Models with Applications to Electrocardiography. In: International Conference on Machine Learning; 2019. p. 4585\u20134594."},{"key":"pcbi.1009439.ref037","unstructured":"Li X, Lin C, Li R, Wang C, Guerin F. Latent space factorisation and manipulation via matrix subspace projection. In: International Conference on Machine Learning. PMLR; 2020. p. 5916\u20135926."},{"issue":"5","key":"pcbi.1009439.ref038","first-page":"6","article-title":"beta-VAE: Learning Basic Visual Concepts with a Constrained Variational Framework","volume":"2","author":"I Higgins","year":"2017","journal-title":"International Conference on Learning Representations"},{"key":"pcbi.1009439.ref039","unstructured":"Kumar A, Sattigeri P, Balakrishnan A. Variational inference of disentangled latent concepts from unlabeled observations. arXiv preprint arXiv:171100848. 2017."},{"issue":"1","key":"pcbi.1009439.ref040","first-page":"1947","article-title":"Emergence of invariance and disentanglement in deep representations","volume":"19","author":"A Achille","year":"2018","journal-title":"The Journal of Machine Learning Research"},{"issue":"12","key":"pcbi.1009439.ref041","doi-asserted-by":"crossref","first-page":"2897","DOI":"10.1109\/TPAMI.2017.2784440","article-title":"Information dropout: Learning optimal representations through noisy computation","volume":"40","author":"A Achille","year":"2018","journal-title":"IEEE transactions on pattern analysis and machine intelligence"},{"key":"pcbi.1009439.ref042","unstructured":"Kim H, Mnih A. Disentangling by factorising. arXiv preprint arXiv:180205983. 2018."},{"key":"pcbi.1009439.ref043","unstructured":"Esmaeili B, Wu H, Jain S, Bozkurt A, Siddharth N, Paige B, et al. Structured disentangled representations. In: The 22nd International Conference on Artificial Intelligence and Statistics; 2019. p. 2525\u20132534."},{"key":"pcbi.1009439.ref044","unstructured":"Gao S, Brekelmans R, Ver Steeg G, Galstyan A. Auto-encoding total correlation explanation. In: The 22nd International Conference on Artificial Intelligence and Statistics; 2019. p. 1157\u20131166."},{"key":"pcbi.1009439.ref045","unstructured":"Chen RT, Li X, Grosse RB, Duvenaud DK. Isolating sources of disentanglement in variational autoencoders. In: Advances in Neural Information Processing Systems; 2018. p. 2610\u20132620."},{"key":"pcbi.1009439.ref046","doi-asserted-by":"crossref","first-page":"e63711","DOI":"10.7554\/eLife.63711","article-title":"Standardized and reproducible measurement of decision-making in mice","volume":"10","author":"International Brain Lab","year":"2021","journal-title":"Elife"},{"issue":"3","key":"pcbi.1009439.ref047","doi-asserted-by":"crossref","first-page":"602","DOI":"10.1016\/j.neuron.2018.03.037","article-title":"Vision and locomotion shape the interactions between neuron types in mouse visual cortex","volume":"98","author":"M Dipoppa","year":"2018","journal-title":"Neuron"},{"key":"pcbi.1009439.ref048","article-title":"Neuroscience cloud analysis as a service","author":"T Abe","year":"2020","journal-title":"bioRxiv"},{"issue":"3","key":"pcbi.1009439.ref049","doi-asserted-by":"crossref","first-page":"482","DOI":"10.1037\/0033-2909.83.3.482","article-title":"The open-field test: a critical review","volume":"83","author":"RN Walsh","year":"1976","journal-title":"Psychological bulletin"},{"key":"pcbi.1009439.ref050","unstructured":"Stringer C. Facemap; 2020. Available from: https:\/\/github.com\/MouseLand\/facemap."},{"issue":"12","key":"pcbi.1009439.ref051","doi-asserted-by":"crossref","first-page":"1846","DOI":"10.1109\/29.45532","article-title":"On the application of hidden Markov models for enhancing noisy speech","volume":"37","author":"Y Ephraim","year":"1989","journal-title":"IEEE Transactions on Acoustics, Speech, and Signal Processing"},{"issue":"7786","key":"pcbi.1009439.ref052","doi-asserted-by":"crossref","first-page":"266","DOI":"10.1038\/s41586-019-1787-x","article-title":"Distributed coding of choice, action and engagement across the mouse brain","volume":"576","author":"NA Steinmetz","year":"2019","journal-title":"Nature"},{"issue":"7124","key":"pcbi.1009439.ref053","doi-asserted-by":"crossref","first-page":"168","DOI":"10.1038\/nature05453","article-title":"Genome-wide atlas of gene expression in the adult mouse brain","volume":"445","author":"ES Lein","year":"2007","journal-title":"Nature"},{"key":"pcbi.1009439.ref054","doi-asserted-by":"crossref","unstructured":"Balntas V, Riba E, Ponsa D, Mikolajczyk K. Learning local feature descriptors with triplets and shallow convolutional neural networks. In: Bmvc. vol. 1; 2016. p. 3.","DOI":"10.5244\/C.30.119"},{"issue":"11","key":"pcbi.1009439.ref055","doi-asserted-by":"crossref","first-page":"1433","DOI":"10.1038\/s41593-020-00706-3","article-title":"Revealing the structure of pharmacobehavioral space through motion sequencing","volume":"23","author":"AB Wiltschko","year":"2020","journal-title":"Nature Neuroscience"},{"key":"pcbi.1009439.ref056","doi-asserted-by":"crossref","first-page":"20","DOI":"10.3389\/fnsys.2019.00020","article-title":"DeepBehavior: A deep learning toolbox for automated analysis of animal and human behavior imaging data","volume":"13","author":"A Arac","year":"2019","journal-title":"Frontiers in systems neuroscience"},{"issue":"2","key":"pcbi.1009439.ref057","doi-asserted-by":"crossref","first-page":"413","DOI":"10.1016\/j.cell.2019.05.023","article-title":"Correlated neural activity across the brains of socially interacting bats","volume":"178","author":"W Zhang","year":"2019","journal-title":"Cell"},{"key":"pcbi.1009439.ref058","article-title":"Simple Behavioral Analysis (SimBA): an open source toolkit for computer classification of complex social behaviors in experimental animals","author":"SR Nilsson","year":"2020","journal-title":"BioRxiv"},{"key":"pcbi.1009439.ref059","doi-asserted-by":"crossref","first-page":"91","DOI":"10.1016\/j.conb.2021.01.008","article-title":"Body language signals for rodent social communication","volume":"68","author":"CL Ebbesen","year":"2021","journal-title":"Current Opinion in Neurobiology"},{"key":"pcbi.1009439.ref060","doi-asserted-by":"crossref","first-page":"e57258","DOI":"10.7554\/eLife.57258","article-title":"A machine-vision approach for automated pain measurement at millisecond timescales","volume":"9","author":"JM Jones","year":"2020","journal-title":"Elife"},{"issue":"6486","key":"pcbi.1009439.ref061","doi-asserted-by":"crossref","first-page":"89","DOI":"10.1126\/science.aaz9468","article-title":"Facial expressions of emotion states and their neuronal correlates in mice","volume":"368","author":"N Dolensek","year":"2020","journal-title":"Science"},{"key":"pcbi.1009439.ref062","first-page":"730218","article-title":"A size principle for leg motor control in Drosophila","author":"AW Azevedo","year":"2019","journal-title":"bioRxiv"},{"issue":"149","key":"pcbi.1009439.ref063","doi-asserted-by":"crossref","DOI":"10.3791\/59979","article-title":"Automated rat single-pellet reaching with 3-dimensional reconstruction of paw and digit trajectories","author":"A Bova","year":"2019","journal-title":"Journal of visualized experiments: JoVE"},{"issue":"1","key":"pcbi.1009439.ref064","doi-asserted-by":"crossref","first-page":"217","DOI":"10.1016\/j.neuron.2019.01.038","article-title":"Spatial and temporal locomotor learning in mouse cerebellum","volume":"102","author":"DM Darmohray","year":"2019","journal-title":"Neuron"},{"issue":"3","key":"pcbi.1009439.ref065","doi-asserted-by":"crossref","first-page":"469","DOI":"10.1016\/j.neuron.2020.07.032","article-title":"Two brain pathways initiate distinct forward walking programs in Drosophila","volume":"108","author":"SS Bidaye","year":"2020","journal-title":"Neuron"},{"key":"pcbi.1009439.ref066","article-title":"The Mouse Action Recognition System (MARS): a software pipeline for automated analysis of social behaviors in mice","author":"C Segalin","year":"2020","journal-title":"bioRxiv"},{"key":"pcbi.1009439.ref067","unstructured":"Locatello F, Bauer S, Lucic M, Raetsch G, Gelly S, Sch\u00f6lkopf B, et al. Challenging common assumptions in the unsupervised learning of disentangled representations. In: International Conference on Machine Learning. PMLR; 2019. p. 4114\u20134124."},{"issue":"7","key":"pcbi.1009439.ref068","doi-asserted-by":"crossref","first-page":"535","DOI":"10.1038\/nmeth.1468","article-title":"Two-photon calcium imaging from head-fixed Drosophila during optomotor walking behavior","volume":"7","author":"JD Seelig","year":"2010","journal-title":"Nature methods"},{"key":"pcbi.1009439.ref069","doi-asserted-by":"crossref","first-page":"e10989","DOI":"10.7554\/eLife.10989","article-title":"Demixed principal component analysis of neural population data","volume":"5","author":"D Kobak","year":"2016","journal-title":"Elife"},{"issue":"1","key":"pcbi.1009439.ref070","doi-asserted-by":"crossref","first-page":"140","DOI":"10.1038\/s41593-020-00733-0","article-title":"Modeling behaviorally relevant neural dynamics enabled by preferential subspace identification","volume":"24","author":"OG Sani","year":"2021","journal-title":"Nature Neuroscience"},{"key":"pcbi.1009439.ref071","unstructured":"Talbot A, Dunson D, Dzirasa K, Carlson D. Supervised Autoencoders Learn Robust Joint Factor Models of Neural Activity. arXiv preprint arXiv:200405209. 2020."},{"key":"pcbi.1009439.ref072","unstructured":"Zhou D, Wei XX. Learning identifiable and interpretable latent models of high-dimensional neural activity using pi-VAE. Advances in Neural Information Processing Systems. 2020;33."},{"issue":"4","key":"pcbi.1009439.ref073","doi-asserted-by":"crossref","first-page":"R53","DOI":"10.1088\/0954-898X_9_4_001","article-title":"A review of methods for spike sorting: the detection and classification of neural action potentials","volume":"9","author":"MS Lewicki","year":"1998","journal-title":"Network: Computation in Neural Systems"},{"key":"pcbi.1009439.ref074","doi-asserted-by":"crossref","first-page":"e67490","DOI":"10.7554\/eLife.67490","article-title":"Non-linear dimensionality reduction on extracellular waveforms reveals cell type diversity in premotor cortex","volume":"10","author":"EK Lee","year":"2021","journal-title":"Elife"},{"key":"pcbi.1009439.ref075","unstructured":"Kingma DP, Mohamed S, Rezende DJ, Welling M. Semi-supervised learning with deep generative models. In: Advances in Neural Information Processing Systems; 2014. p. 3581\u20133589."},{"key":"pcbi.1009439.ref076","unstructured":"Sohn K, Lee H, Yan X. Learning structured output representation using deep conditional generative models. In: Advances in Neural Information Processing Systems; 2015. p. 3483\u20133491."},{"key":"pcbi.1009439.ref077","unstructured":"Perarnau G, Van De Weijer J, Raducanu B, \u00c1lvarez JM. Invertible conditional gans for image editing. arXiv preprint arXiv:161106355. 2016."},{"key":"pcbi.1009439.ref078","doi-asserted-by":"crossref","unstructured":"Yan X, Yang J, Sohn K, Lee H. Attribute2image: Conditional image generation from visual attributes. In: European Conference on Computer Vision. Springer; 2016. p. 776\u2013791.","DOI":"10.1007\/978-3-319-46493-0_47"},{"key":"pcbi.1009439.ref079","unstructured":"Creswell A, Mohamied Y, Sengupta B, Bharath AA. Adversarial information factorization. arXiv preprint arXiv:171105175. 2017."},{"key":"pcbi.1009439.ref080","unstructured":"Lample G, Zeghidour N, Usunier N, Bordes A, Denoyer L, Ranzato M. Fader networks: Manipulating images by sliding attributes. In: Advances in Neural Information Processing Systems; 2017. p. 5967\u20135976."},{"key":"pcbi.1009439.ref081","unstructured":"Klys J, Snell J, Zemel R. Learning latent subspaces in variational autoencoders. In: Advances in Neural Information Processing Systems; 2018. p. 6444\u20136454."},{"key":"pcbi.1009439.ref082","unstructured":"Khemakhem I, Kingma D, Monti R, Hyvarinen A. Variational autoencoders and nonlinear ica: A unifying framework. In: International Conference on Artificial Intelligence and Statistics; 2020. p. 2207\u20132217."},{"key":"pcbi.1009439.ref083","unstructured":"Kumar M, Babaeizadeh M, Erhan D, Finn C, Levine S, Dinh L, et al. VideoFlow: A conditional flow-based model for stochastic video generation. arXiv preprint arXiv:190301434. 2019."},{"key":"pcbi.1009439.ref084","unstructured":"Klindt D, Schott L, Sharma Y, Ustyuzhaninov I, Brendel W, Bethge M, et al. Towards Nonlinear Disentanglement in Natural Data with Temporal Sparse Coding. arXiv preprint arXiv:200710930. 2020."},{"key":"pcbi.1009439.ref085","first-page":"802","article-title":"Convolutional LSTM network: A machine learning approach for precipitation nowcasting","volume":"28","author":"X Shi","year":"2015","journal-title":"Advances in Neural Information Processing Systems"},{"key":"pcbi.1009439.ref086","unstructured":"Babaeizadeh M, Finn C, Erhan D, Campbell RH, Levine S. Stochastic variational video prediction. arXiv preprint arXiv:171011252. 2017."},{"key":"pcbi.1009439.ref087","unstructured":"Denton E, Fergus R. Stochastic video generation with a learned prior. arXiv preprint arXiv:180207687. 2018."},{"key":"pcbi.1009439.ref088","unstructured":"Lee AX, Zhang R, Ebert F, Abbeel P, Finn C, Levine S. Stochastic adversarial video prediction. arXiv preprint arXiv:180401523. 2018."},{"key":"pcbi.1009439.ref089","doi-asserted-by":"crossref","unstructured":"Castrejon L, Ballas N, Courville A. Improved conditional vrnns for video prediction. In: Proceedings of the IEEE International Conference on Computer Vision; 2019. p. 7608\u20137617.","DOI":"10.1109\/ICCV.2019.00770"},{"key":"pcbi.1009439.ref090","unstructured":"Pearce M. The Gaussian Process Prior VAE for Interpretable Latent Dynamics from Pixels. In: Symposium on Advances in Approximate Bayesian Inference; 2020. p. 1\u201312."},{"key":"pcbi.1009439.ref091","unstructured":"Larsen ABL, S\u00f8nderby SK, Larochelle H, Winther O. Autoencoding beyond pixels using a learned similarity metric. arXiv preprint arXiv:151209300. 2015."},{"key":"pcbi.1009439.ref092","unstructured":"Chen X, Duan Y, Houthooft R, Schulman J, Sutskever I, Abbeel P. Infogan: Interpretable representation learning by information maximizing generative adversarial nets. In: Advances in Neural Information Processing Systems; 2016. p. 2172\u20132180."},{"key":"pcbi.1009439.ref093","unstructured":"Zhao S, Song J, Ermon S. Infovae: Information maximizing variational autoencoders. arXiv preprint arXiv:170602262. 2017."},{"key":"pcbi.1009439.ref094","article-title":"Suite2p: beyond 10,000 neurons with standard two-photon microscopy","author":"M Pachitariu","year":"2017","journal-title":"Biorxiv"},{"key":"pcbi.1009439.ref095","doi-asserted-by":"crossref","unstructured":"Churchland AK, Musall S, Kaufmann MT, Juavinett AL, Gluf S. Single-trial neural dynamics are dominated by richly varied movements:dataset. 2019; https:\/\/dx.doi.org\/10.14224\/1.38599.","DOI":"10.14224\/1.38599"},{"key":"pcbi.1009439.ref096","unstructured":"Titsias M, L\u00e1zaro-Gredilla M. Doubly stochastic variational Bayes for non-conjugate inference. In: International Conference on Machine Learning; 2014. p. 1971\u20131979."},{"key":"pcbi.1009439.ref097","unstructured":"Hoffman MD, Johnson MJ. Elbo surgery: yet another way to carve up the variational evidence lower bound. In: Workshop in Advances in Approximate Bayesian Inference, NIPS. vol. 1; 2016. p. 2."},{"key":"pcbi.1009439.ref098","unstructured":"Kingma DP, Ba J. Adam: A method for stochastic optimization. arXiv preprint arXiv:14126980. 2014."},{"key":"pcbi.1009439.ref099","doi-asserted-by":"crossref","unstructured":"Bowman SR, Vilnis L, Vinyals O, Dai AM, Jozefowicz R, Bengio S. Generating sentences from a continuous space. arXiv preprint arXiv:151106349. 2015.","DOI":"10.18653\/v1\/K16-1002"},{"key":"pcbi.1009439.ref100","unstructured":"Linderman S. SSM: Bayesian learning and inference for state space models; 2020. Available from: https:\/\/github.com\/lindermanlab\/ssm."}],"updated-by":[{"DOI":"10.1371\/journal.pcbi.1009439","type":"new_version","label":"New version","source":"publisher","updated":{"date-parts":[[2021,10,4]],"date-time":"2021-10-04T00:00:00Z","timestamp":1633305600000}}],"container-title":["PLOS Computational Biology"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pcbi.1009439","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,10,4]],"date-time":"2021-10-04T18:31:17Z","timestamp":1633372277000},"score":1,"resource":{"primary":{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pcbi.1009439"}},"subtitle":[],"editor":[{"given":"Fr\u00e9d\u00e9ric E.","family":"Theunissen","sequence":"first","affiliation":[]}],"short-title":[],"issued":{"date-parts":[[2021,9,22]]},"references-count":100,"journal-issue":{"issue":"9","published-online":{"date-parts":[[2021,9,22]]}},"URL":"https:\/\/doi.org\/10.1371\/journal.pcbi.1009439","relation":{"has-preprint":[{"id-type":"doi","id":"10.1101\/2021.02.22.432309","asserted-by":"object"}]},"ISSN":["1553-7358"],"issn-type":[{"value":"1553-7358","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,9,22]]}}}