{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,9]],"date-time":"2026-05-09T17:06:25Z","timestamp":1778346385079,"version":"3.51.4"},"reference-count":56,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2022,4,21]],"date-time":"2022-04-21T00:00:00Z","timestamp":1650499200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,4,21]],"date-time":"2022-04-21T00:00:00Z","timestamp":1650499200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nat Mach Intell"],"DOI":"10.1038\/s42256-022-00477-5","type":"journal-article","created":{"date-parts":[[2022,4,21]],"date-time":"2022-04-21T23:03:53Z","timestamp":1650582233000},"page":"331-340","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":113,"title":["Deep-learning-based identification, tracking, pose estimation and behaviour classification of interacting primates and mice in complex environments"],"prefix":"10.1038","volume":"4","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8016-1637","authenticated-orcid":false,"given":"Markus","family":"Marks","sequence":"first","affiliation":[]},{"given":"Qiuhan","family":"Jin","sequence":"additional","affiliation":[]},{"given":"Oliver","family":"Sturman","sequence":"additional","affiliation":[]},{"given":"Lukas","family":"von Ziegler","sequence":"additional","affiliation":[]},{"given":"Sepp","family":"Kollmorgen","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6883-0236","authenticated-orcid":false,"given":"Wolfger","family":"von der Behrens","sequence":"additional","affiliation":[]},{"given":"Valerio","family":"Mante","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8442-653X","authenticated-orcid":false,"given":"Johannes","family":"Bohacek","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8963-2893","authenticated-orcid":false,"given":"Mehmet Fatih","family":"Yanik","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2022,4,21]]},"reference":[{"key":"477_CR1","doi-asserted-by":"publisher","first-page":"11","DOI":"10.1016\/j.neuron.2019.09.038","volume":"104","author":"SR Datta","year":"2019","unstructured":"Datta, S. R., Anderson, D. J., Branson, K., Perona, P. & Leifer, A. Computational neuroethology: a call to action. Neuron 104, 11\u201324 (2019).","journal-title":"Neuron"},{"key":"477_CR2","doi-asserted-by":"crossref","unstructured":"Mathis, A. et al. DeepLabCut: markerless pose estimation of user-defined body parts with deep learning. Nature 21, 1281\u20131289 (2018).","DOI":"10.1038\/s41593-018-0209-y"},{"key":"477_CR3","doi-asserted-by":"publisher","first-page":"124","DOI":"10.1038\/s42003-019-0362-1","volume":"2","author":"BQ Geuther","year":"2019","unstructured":"Geuther, B. Q. et al. Robust mouse tracking in complex environments using neural networks. Commun. Biol. 2, 124 (2019).","journal-title":"Commun. Biol."},{"key":"477_CR4","doi-asserted-by":"publisher","first-page":"179","DOI":"10.1038\/s41592-018-0295-5","volume":"16","author":"F Romero-Ferrero","year":"2019","unstructured":"Romero-Ferrero, F., Bergomi, M. G., Hinz, R. C., Heras, F. J. & de Polavieja, G. idtracker.ai: Tracking all individuals in small or large collectives of unmarked animals. Nat. Methods 16, 179 (2019).","journal-title":"Nat. Methods"},{"key":"477_CR5","doi-asserted-by":"crossref","unstructured":"Forys, B. J., Xiao, D., Gupta, P. & Murphy, T. H. Real-time selective markerless tracking of forepaws of head fixed mice using deep neural networks. eNeuro 7, ENEURO.0096-20.2020 (2020).","DOI":"10.1523\/ENEURO.0096-20.2020"},{"key":"477_CR6","doi-asserted-by":"publisher","first-page":"117","DOI":"10.1038\/s41592-018-0234-5","volume":"16","author":"TD Pereira","year":"2019","unstructured":"Pereira, T. D. et al. Fast animal pose estimation using deep neural networks. Nat. Methods 16, 117 (2019).","journal-title":"Nat. Methods"},{"key":"477_CR7","doi-asserted-by":"publisher","first-page":"e47994","DOI":"10.7554\/eLife.47994","volume":"8","author":"JM Graving","year":"2019","unstructured":"Graving, J. M. et al. DeepPoseKit, a software toolkit for fast and robust animal pose estimation using deep learning. eLife 8, e47994 (2019).","journal-title":"eLife"},{"key":"477_CR8","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-020-18441-5","volume":"11","author":"PC Bala","year":"2020","unstructured":"Bala, P. C. et al. Automated markerless pose estimation in freely moving macaques with OpenMonkeyStudio. Nat. Commun. 11, 4560 (2020).","journal-title":"Nat. Commun."},{"key":"477_CR9","doi-asserted-by":"publisher","first-page":"e48571","DOI":"10.7554\/eLife.48571","volume":"8","author":"S G\u00fcnel","year":"2019","unstructured":"G\u00fcnel, S. et al. DeepFly3D, a deep learning-based approach for 3D limb and appendage tracking in tethered, adult Drosophila. eLife 8, e48571 (2019).","journal-title":"eLife"},{"key":"477_CR10","doi-asserted-by":"crossref","unstructured":"Chen, Z. et al. AlphaTracker: a multi-animal tracking and behavioral analysis tool. Preprint at https:\/\/www.biorxiv.org\/content\/10.1101\/2020.12.04.405159v1 (2020).","DOI":"10.1101\/2020.12.04.405159"},{"key":"477_CR11","doi-asserted-by":"crossref","unstructured":"Lauer, J. et al. Multi-animal pose estimation and tracking with DeepLabCut. Preprint at https:\/\/www.biorxiv.org\/content\/10.1101\/2021.04.30.442096v1 (2021).","DOI":"10.1101\/2021.04.30.442096"},{"key":"477_CR12","doi-asserted-by":"publisher","first-page":"1121","DOI":"10.1016\/j.neuron.2015.11.031","volume":"88","author":"AB Wiltschko","year":"2015","unstructured":"Wiltschko, A. B. et al. Mapping sub-second structure in mouse behavior. Neuron 88, 1121\u20131135 (2015).","journal-title":"Neuron"},{"key":"477_CR13","doi-asserted-by":"crossref","unstructured":"Hsu, A. I. & Yttri, E. A. B-SOiD: an open source unsupervised algorithm for discovery of spontaneous behaviors. Nat Commun. 12, 5188 (2019).","DOI":"10.1101\/770271"},{"key":"477_CR14","doi-asserted-by":"publisher","first-page":"20140672","DOI":"10.1098\/rsif.2014.0672","volume":"11","author":"GJ Berman","year":"2014","unstructured":"Berman, G. J., Choi, D. M., Bialek, W. & Shaevitz, J. W. Mapping the stereotyped behaviour of freely moving fruit flies. J. R. Soc. Interface 11, 20140672 (2014).","journal-title":"J. R. Soc. Interface"},{"key":"477_CR15","doi-asserted-by":"publisher","first-page":"e1009439","DOI":"10.1371\/journal.pcbi.1009439","volume":"17","author":"MR Whiteway","year":"2021","unstructured":"Whiteway, M. R. et al. Partitioning variability in animal behavioral videos using semi-supervised variational autoencoders. PLoS Comput. Biol. 17, e1009439 (2021).","journal-title":"PLoS Comput. Biol."},{"key":"477_CR16","doi-asserted-by":"publisher","first-page":"2040","DOI":"10.1038\/s41593-019-0533-x","volume":"22","author":"AJ Calhoun","year":"2019","unstructured":"Calhoun, A. J., Pillow, J. W. & Murthy, M. Unsupervised identification of the internal states that shape natural behavior. Nat. Neurosci. 22, 2040\u20132049 (2019).","journal-title":"Nat. Neurosci."},{"key":"477_CR17","unstructured":"Batty, E. et al. BehaveNet: Nonlinear Embedding and Bayesian Neural Decoding of Behavioral Videos (NeurIPS, 2019)."},{"key":"477_CR18","doi-asserted-by":"crossref","unstructured":"Nilsson, S. R. et al. Simple behavioral analysis (SimBA)\u2014an open source toolkit for computer classification of complex social behaviors in experimental animals. Preprint at https:\/\/www.biorxiv.org\/content\/10.1101\/2020.04.19.049452v2 (2020).","DOI":"10.1101\/2020.04.19.049452"},{"key":"477_CR19","doi-asserted-by":"publisher","first-page":"e63720","DOI":"10.7554\/eLife.63720","volume":"10","author":"C Segalin","year":"2021","unstructured":"Segalin, C. et al. The Mouse Action Recognition System (MARS) software pipeline for automated analysis of social behaviors in mice. eLife 10, e63720 (2021).","journal-title":"eLife"},{"key":"477_CR20","doi-asserted-by":"crossref","unstructured":"Sturman, O. et al. Deep learning-based behavioral analysis reaches human accuracy and is capable of outperforming commercial solutions. Neuropsychopharmacology 45, 1942\u20131952 (2020).","DOI":"10.1038\/s41386-020-0776-y"},{"key":"477_CR21","doi-asserted-by":"publisher","first-page":"1052","DOI":"10.1038\/s41592-020-0961-2","volume":"17","author":"A Nourizonoz","year":"2020","unstructured":"Nourizonoz, A. et al. EthoLoop: automated closed-loop neuroethology in naturalistic environments. Nat. Methods 17, 1052\u20131059 (2020).","journal-title":"Nat. Methods"},{"key":"477_CR22","doi-asserted-by":"publisher","first-page":"451","DOI":"10.1038\/nmeth.1328","volume":"6","author":"K Branson","year":"2009","unstructured":"Branson, K., Robie, A. A., Bender, J., Perona, P. & Dickinson, M. H. High-throughput ethomics in large groups of Drosophila. Nat. Methods 6, 451\u2013457 (2009).","journal-title":"Nat. Methods"},{"key":"477_CR23","doi-asserted-by":"publisher","first-page":"297","DOI":"10.1038\/nmeth.1310","volume":"6","author":"H Dankert","year":"2009","unstructured":"Dankert, H., Wang, L., Hoopfer, E. D., Anderson, D. J. & Perona, P. Automated monitoring and analysis of social behavior in Drosophila. Nat. Methods 6, 297\u2013303 (2009).","journal-title":"Nat. Methods"},{"key":"477_CR24","doi-asserted-by":"publisher","first-page":"64","DOI":"10.1038\/nmeth.2281","volume":"10","author":"M Kabra","year":"2013","unstructured":"Kabra, M., Robie, A. A., Rivera-Alba, M., Branson, S. & Branson, K. JAABA: interactive machine learning for automatic annotation of animal behavior. Nat. Methods 10, 64 (2013).","journal-title":"Nat. Methods"},{"key":"477_CR25","doi-asserted-by":"publisher","DOI":"10.1038\/ncomms1064","volume":"1","author":"H Jhuang","year":"2010","unstructured":"Jhuang, H. et al. Automated home-cage behavioural phenotyping of mice. Nat. Commun. 1, 68 (2010).","journal-title":"Nat. Commun."},{"key":"477_CR26","doi-asserted-by":"publisher","unstructured":"Hayden, B. Y., Park, H. S. & Zimmermann, J. Automated pose estimation in primates. Am. J. Primatol. https:\/\/doi.org\/10.1002\/ajp.23348 (2021).","DOI":"10.1002\/ajp.23348"},{"key":"477_CR27","doi-asserted-by":"crossref","unstructured":"He, K., Gkioxari, G., Doll\u00e1r, P. & Girshick, R. Mask R-CNN. In Proc. IEEE International Conference on Computer Vision 2961\u20132969 (IEEE, 2017).","DOI":"10.1109\/ICCV.2017.322"},{"key":"477_CR28","doi-asserted-by":"crossref","unstructured":"Huang, G., Liu, Z., Van Der Maaten, L. & Weinberger, K. Q. Densely connected convolutional networks. In Proc. IEEE Conference on Computer Vision and Pattern Recognition 4700\u20134708 (IEEE, 2017).","DOI":"10.1109\/CVPR.2017.243"},{"key":"477_CR29","doi-asserted-by":"crossref","unstructured":"Cho, K. et al. Learning phrase representations using RNN encoder\u2013decoder for statistical machine translation. In Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP) 1724\u20131734 (Association for Computational Linguistics, 2014).","DOI":"10.3115\/v1\/D14-1179"},{"key":"477_CR30","unstructured":"Chung, J., Gulcehre, C., Cho, K. & Bengio, Y. Empirical evaluation of gated recurrent neural networks on sequence modeling. In NeurIPS 2014 Workshop on Deep Learning (2014)."},{"key":"477_CR31","doi-asserted-by":"crossref","unstructured":"Deb, D. et al. Face recognition: primates in the wild. Preprint at https:\/\/arxiv.org\/abs\/1804.08790 (2018).","DOI":"10.1109\/BTAS.2018.8698538"},{"key":"477_CR32","doi-asserted-by":"crossref","unstructured":"Chollet, F. Xception: deep learning with depthwise separable convolutions. In Proc. IEEE Conference on Computer Vision and Pattern Recognition 1251\u20131258 (IEEE, 2017).","DOI":"10.1109\/CVPR.2017.195"},{"key":"477_CR33","unstructured":"Van den Oord, A. et al. WaveNet: a generative model for raw audio. Preprint at https:\/\/arxiv.org\/abs\/1609.03499 (2016)"},{"key":"477_CR34","unstructured":"Bai, S., Kolter, J. Z. & Koltun, V. An empirical evaluation of generic convolutional and recurrent networks for sequence modeling. Preprint at https:\/\/arxiv.org\/abs\/1803.01271 (2018)."},{"key":"477_CR35","unstructured":"Jung, A. B. et al. Imgaug (GitHub, 2020); https:\/\/github.com\/aleju\/imgaug"},{"key":"477_CR36","unstructured":"Yosinski, J., Clune, J., Bengio, Y. & Lipson, H. How transferable are features in deep neural networks? In Advances in Neural Information Processing Systems 3320\u20133328 (NeurIPS, 2014)."},{"key":"477_CR37","doi-asserted-by":"crossref","unstructured":"He, K., Zhang, X., Ren, S. & Sun, J. Deep residual learning for image recognition. In Proc. IEEE Conference on Computer Vision and Pattern Recognition 770\u2013778 (IEEE, 2016).","DOI":"10.1109\/CVPR.2016.90"},{"key":"477_CR38","doi-asserted-by":"crossref","unstructured":"Lin, T.-Y. et al. Microsoft COCO: Common Objects in Context. In European Conference on Computer Vision 740\u2013755 (Springer, 2014).","DOI":"10.1007\/978-3-319-10602-1_48"},{"key":"477_CR39","doi-asserted-by":"publisher","unstructured":"Dutta, A. & Zisserman, A. The VIA annotation software for images, audio and video. In Proc. 27th ACM International Conference on Multimedia (ACM, 2019); https:\/\/doi.org\/10.1145\/3343031.3350535","DOI":"10.1145\/3343031.3350535"},{"key":"477_CR40","doi-asserted-by":"crossref","unstructured":"Xiao, B., Wu, H. & Wei, Y. Simple baselines for human pose estimation and tracking. In Computer Vision \u2013 ECCV 2018 (eds. Ferrari, V., Hebert, M., Sminchisescu, C. & Weiss, Y.) 472\u2013487 (Springer International Publishing, 2018).","DOI":"10.1007\/978-3-030-01231-1_29"},{"key":"477_CR41","unstructured":"Tan, M. & Le, Q. V. EfficientNet: rethinking model scaling for convolutional neural networks. Preprint at https:\/\/arxiv.org\/abs\/1905.11946 (2020)."},{"key":"477_CR42","unstructured":"Krizhevsky, A., Sutskever, I. & Hinton, G. E. Imagenet classification with deep convolutional neural networks. In Advances in Neural Information Processing Systems 1097\u20131105 (NeurIPS, 2012)."},{"key":"477_CR43","doi-asserted-by":"publisher","first-page":"900","DOI":"10.1093\/icb\/icab107","volume":"61","author":"M Vidal","year":"2021","unstructured":"Vidal, M., Wolf, N., Rosenberg, B., Harris, B. P. & Mathis, A. Perspectives on individual animal identification from biology and computer vision. Integr. Comp. Biol. 61, 900\u2013916 (2021).","journal-title":"Integr. Comp. Biol."},{"key":"477_CR44","first-page":"1","volume":"7","author":"J Dem\u0161ar","year":"2006","unstructured":"Dem\u0161ar, J. Statistical comparisons of classifiers over multiple data sets. J. Mach. Learn. Res. 7, 1\u201330 (2006).","journal-title":"J. Mach. Learn. Res."},{"key":"477_CR45","doi-asserted-by":"publisher","first-page":"2319","DOI":"10.1126\/science.290.5500.2319","volume":"290","author":"JB Tenenbaum","year":"2000","unstructured":"Tenenbaum, J. B. A global geometric framework for nonlinear dimensionality reduction. Science 290, 2319\u20132323 (2000).","journal-title":"Science"},{"key":"477_CR46","doi-asserted-by":"publisher","unstructured":"Lin, T.-Y. et al. Feature pyramid networks for object detection. In 2017 IEEE Conference on Computer Vision and Pattern Recognition (CVPR) 936\u2013944 (IEEE, 2017); https:\/\/doi.org\/10.1109\/CVPR.2017.106","DOI":"10.1109\/CVPR.2017.106"},{"key":"477_CR47","doi-asserted-by":"publisher","unstructured":"Girshick, R. Fast R-CNN. In 2015 IEEE International Conference on Computer Vision (ICCV) 1440\u20131448 (IEEE, 2015); https:\/\/doi.org\/10.1109\/ICCV.2015.169","DOI":"10.1109\/ICCV.2015.169"},{"key":"477_CR48","unstructured":"Srivastava, N., Hinton, G., Krizhevsky, A., Sutskever, I. & Salakhutdinov, R. Dropout: a simple way to prevent neural networks from overfitting. J. Mach. Learn. Res. 15, 1929\u22121958 (2014)."},{"key":"477_CR49","unstructured":"Ioffe, S. & Szegedy, C. Batch normalization: accelerating deep network training by reducing internal covariate shift. In Proceedings of the 32nd International Conference on International Conference on Machine Learning Vol. 37, 448\u2013456 (JMLR.org, 2015)."},{"key":"477_CR50","unstructured":"Maas, A. L., Hannun, A. Y. & Ng, A. Y. Rectifier nonlinearities improve neural network acoustic models. In Proc. 30th International Conference on Machine Learning (ICML, 2013)."},{"key":"477_CR51","unstructured":"Xu, B., Wang, N., Chen, T. & Li, M. Empirical evaluation of rectified activations in convolutional network. Preprint at https:\/\/arxiv.org\/abs\/1505.00853 (2015)."},{"key":"477_CR52","unstructured":"Kingma, D. P. & Ba, J. Adam: a method for stochastic optimization. In Proceedings of the 3rd International Conference on Learning Representations (ICLR, 2014)."},{"key":"477_CR53","doi-asserted-by":"crossref","unstructured":"Lin, T.-Y., Goyal, P., Girshick, R., He, K. & Dollar, P. Focal loss for Dense object detection. In 2017 IEEE International Conference on Computer Vision (ICCV, 2017).","DOI":"10.1109\/ICCV.2017.324"},{"key":"477_CR54","doi-asserted-by":"crossref","unstructured":"Bohnslav, J. P. et al. DeepEthogram: a machine learning pipeline for supervised behavior classification from raw pixels. eLife 10, 63377 (2020).","DOI":"10.7554\/eLife.63377"},{"key":"477_CR55","unstructured":"Abadi, M. et al. TensorFlow: large-scale machine learning on heterogeneous distributed systems. Preprint at https:\/\/arxiv.org\/abs\/1603.04467 (2016)."},{"key":"477_CR56","unstructured":"Chollet, F. Keras (GitHub, 2015); https:\/\/github.com\/fchollet\/keras"}],"container-title":["Nature Machine Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s42256-022-00477-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s42256-022-00477-5","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s42256-022-00477-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,11,25]],"date-time":"2022-11-25T09:27:50Z","timestamp":1669368470000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s42256-022-00477-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,4,21]]},"references-count":56,"journal-issue":{"issue":"4","published-online":{"date-parts":[[2022,4]]}},"alternative-id":["477"],"URL":"https:\/\/doi.org\/10.1038\/s42256-022-00477-5","relation":{},"ISSN":["2522-5839"],"issn-type":[{"value":"2522-5839","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,4,21]]},"assertion":[{"value":"25 October 2020","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 March 2022","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"21 April 2022","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}