{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,2]],"date-time":"2026-04-02T02:18:27Z","timestamp":1775096307597,"version":"3.50.1"},"update-to":[{"DOI":"10.1371\/journal.pcbi.1006518","type":"new_version","label":"New version","source":"publisher","updated":{"date-parts":[[2018,11,6]],"date-time":"2018-11-06T00:00:00Z","timestamp":1541462400000}}],"reference-count":61,"publisher":"Public Library of Science (PLoS)","issue":"10","license":[{"start":{"date-parts":[[2018,10,25]],"date-time":"2018-10-25T00:00:00Z","timestamp":1540425600000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["CNS 1624378"],"award-info":[{"award-number":["CNS 1624378"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["EY05729"],"award-info":[{"award-number":["EY05729"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100006785","name":"Google","doi-asserted-by":"publisher","award":["AR\/VR Research Award"],"award-info":[{"award-number":["AR\/VR Research Award"]}],"id":[{"id":"10.13039\/100006785","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["www.ploscompbiol.org"],"crossmark-restriction":false},"short-container-title":["PLoS Comput Biol"],"DOI":"10.1371\/journal.pcbi.1006518","type":"journal-article","created":{"date-parts":[[2018,10,25]],"date-time":"2018-10-25T13:29:53Z","timestamp":1540474193000},"page":"e1006518","update-policy":"https:\/\/doi.org\/10.1371\/journal.pcbi.corrections_policy","source":"Crossref","is-referenced-by-count":13,"title":["Modeling sensory-motor decisions in natural behavior"],"prefix":"10.1371","volume":"14","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6681-3360","authenticated-orcid":true,"given":"Ruohan","family":"Zhang","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Shun","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Matthew H.","family":"Tong","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yuchen","family":"Cui","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Constantin A.","family":"Rothkopf","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dana H.","family":"Ballard","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mary M.","family":"Hayhoe","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"340","published-online":{"date-parts":[[2018,10,25]]},"reference":[{"key":"ref1","doi-asserted-by":"crossref","first-page":"389","DOI":"10.1146\/annurev-vision-102016-061437","article-title":"Vision and action","volume":"3","author":"MM Hayhoe","year":"2017","journal-title":"Annual review of vision science"},{"issue":"2","key":"ref2","doi-asserted-by":"crossref","first-page":"11","DOI":"10.1145\/1265957.1265960","article-title":"Modeling embodied visual behaviors","volume":"4","author":"N Sprague","year":"2007","journal-title":"ACM Transactions on Applied Perception (TAP)"},{"issue":"14","key":"ref3","doi-asserted-by":"crossref","first-page":"16","DOI":"10.1167\/7.14.16","article-title":"Task and context determine where you look","volume":"7","author":"CA Rothkopf","year":"2007","journal-title":"Journal of vision"},{"key":"ref4","article-title":"Control of gaze while walking: task structure, reward, and uncertainty","author":"MH Tong","year":"2017","journal-title":"Journal of Vision"},{"key":"ref5","doi-asserted-by":"crossref","DOI":"10.1109\/TNN.1998.712192","article-title":"Introduction to reinforcement learning","author":"RS Sutton","year":"1998"},{"issue":"6","key":"ref6","doi-asserted-by":"crossref","first-page":"996","DOI":"10.1016\/j.conb.2012.05.003","article-title":"Motor control is decision-making","volume":"22","author":"DM Wolpert","year":"2012","journal-title":"Current opinion in neurobiology"},{"issue":"7","key":"ref7","doi-asserted-by":"crossref","first-page":"1660","DOI":"10.1523\/JNEUROSCI.3417-03.2004","article-title":"A neural correlate of reward-based behavioral learning in caudate nucleus: a functional magnetic resonance imaging study of a stochastic decision task","volume":"24","author":"M Haruno","year":"2004","journal-title":"The Journal of Neuroscience"},{"issue":"4","key":"ref8","doi-asserted-by":"crossref","first-page":"679","DOI":"10.1037\/0033-295X.109.4.679","article-title":"The neural basis of human error processing: reinforcement learning, dopamine, and the error-related negativity","volume":"109","author":"CB Holroyd","year":"2002","journal-title":"Psychological review"},{"issue":"2","key":"ref9","doi-asserted-by":"crossref","first-page":"205","DOI":"10.1016\/j.conb.2007.03.004","article-title":"Efficient reinforcement learning: computational theories, neuroscience and robotics","volume":"17","author":"M Kawato","year":"2007","journal-title":"Current opinion in neurobiology"},{"issue":"1","key":"ref10","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1002\/(SICI)1098-1063(2000)10:1<1::AID-HIPO1>3.0.CO;2-1","article-title":"A model of hippocampally dependent navigation, using the temporal difference learning rule","volume":"10","author":"D Foster","year":"2000","journal-title":"Hippocampus"},{"key":"ref11","doi-asserted-by":"crossref","first-page":"287","DOI":"10.1146\/annurev-neuro-062111-150512","article-title":"Neural basis of reinforcement learning and decision making","volume":"35","author":"D Lee","year":"2012","journal-title":"Annual review of neuroscience"},{"issue":"8","key":"ref12","doi-asserted-by":"crossref","first-page":"1277","DOI":"10.1016\/j.neunet.2006.03.004","article-title":"Neural systems implicated in delayed and probabilistic reinforcement","volume":"19","author":"RN Cardinal","year":"2006","journal-title":"Neural Networks"},{"issue":"6","key":"ref13","doi-asserted-by":"crossref","first-page":"1204","DOI":"10.1016\/j.neuron.2011.02.027","article-title":"Model-based influences on humans\u2019 choices and striatal prediction errors","volume":"69","author":"ND Daw","year":"2011","journal-title":"Neuron"},{"issue":"9","key":"ref14","doi-asserted-by":"crossref","first-page":"680","DOI":"10.1038\/s41562-017-0180-8","article-title":"The successor representation in human reinforcement learning","volume":"1","author":"I Momennejad","year":"2017","journal-title":"Nature Human Behaviour"},{"issue":"4","key":"ref15","doi-asserted-by":"crossref","first-page":"410","DOI":"10.1038\/nn2077","article-title":"Modulators of decision making","volume":"11","author":"K Doya","year":"2008","journal-title":"Nature neuroscience"},{"issue":"8","key":"ref16","doi-asserted-by":"crossref","first-page":"887","DOI":"10.1038\/nn1279","article-title":"Prediction of immediate and future rewards differentially recruits cortico-basal ganglia loops","volume":"7","author":"SC Tanaka","year":"2004","journal-title":"Nature neuroscience"},{"issue":"13","key":"ref17","doi-asserted-by":"crossref","first-page":"R622","DOI":"10.1016\/j.cub.2014.05.020","article-title":"Modeling task control of eye movements","volume":"24","author":"M Hayhoe","year":"2014","journal-title":"Current Biology"},{"key":"ref18","unstructured":"Ng AY, Russell SJ. Algorithms for Inverse Reinforcement Learning. In: Proceedings of the Seventeenth International Conference on Machine Learning. Morgan Kaufmann Publishers Inc.; 2000. p. 663\u2013670."},{"issue":"4","key":"ref19","doi-asserted-by":"crossref","first-page":"477","DOI":"10.1007\/s00422-013-0562-6","article-title":"Modular inverse reinforcement learning for visuomotor behavior","volume":"107","author":"CA Rothkopf","year":"2013","journal-title":"Biological cybernetics"},{"issue":"7","key":"ref20","doi-asserted-by":"crossref","first-page":"985","DOI":"10.1016\/S0893-6080(02)00235-6","article-title":"Inter-module credit assignment in modular reinforcement learning","volume":"16","author":"K Samejima","year":"2003","journal-title":"Neural Networks"},{"key":"ref21","unstructured":"Sprague N, Ballard D. Multiple-goal reinforcement learning with modular Sarsa (O). In: Proceedings of the 18th international joint conference on Artificial intelligence. Morgan Kaufmann Publishers Inc.; 2003. p. 1445\u20131447."},{"key":"ref22","doi-asserted-by":"crossref","first-page":"177","DOI":"10.1163\/22134808-00002414","article-title":"A hierarchical modular architecture for embodied cognition","volume":"26","author":"DH Ballard","year":"2013","journal-title":"Multisensory research"},{"issue":"43","key":"ref23","doi-asserted-by":"crossref","first-page":"13524","DOI":"10.1523\/JNEUROSCI.2469-09.2009","article-title":"Human reinforcement learning subdivides structured action spaces by learning effector-specific values","volume":"29","author":"SJ Gershman","year":"2009","journal-title":"The Journal of Neuroscience"},{"issue":"17","key":"ref24","doi-asserted-by":"crossref","first-page":"4528","DOI":"10.1523\/JNEUROSCI.4982-07.2008","article-title":"Low-serotonin levels increase delayed reward discounting in humans","volume":"28","author":"N Schweighofer","year":"2008","journal-title":"the Journal of Neuroscience"},{"key":"ref25","doi-asserted-by":"crossref","DOI":"10.3389\/fnbeh.2014.00076","article-title":"Does temporal discounting explain unhealthy behavior? A systematic review and reinforcement learning perspective","volume":"8","author":"GW Story","year":"2014","journal-title":"Frontiers in behavioral neuroscience"},{"issue":"12","key":"ref26","doi-asserted-by":"crossref","first-page":"e783","DOI":"10.1167\/15.12.783","article-title":"Objects in the peripheral visual field influence gaze location in natural vision","volume":"15","author":"E Hitzel","year":"2015","journal-title":"Journal of vision"},{"issue":"01","key":"ref27","doi-asserted-by":"crossref","first-page":"81","DOI":"10.1017\/S0952523808080978","article-title":"Image statistics at the point of gaze during human navigation","volume":"26","author":"CA Rothkopf","year":"2009","journal-title":"Visual neuroscience"},{"key":"ref28","article-title":"Multitask Human Navigation in VR with Motion Tracking","author":"MH Tong","year":"2017","journal-title":"Multitask Human Navigation in VR with Motion Tracking"},{"key":"ref29","unstructured":"Russell SJ, Zimdars A. Q-Decomposition for Reinforcement Learning Agents. In: Proceedings of the 20th International Conference on Machine Learning (ICML-03); 2003. p. 656\u2013663."},{"key":"ref30","doi-asserted-by":"crossref","first-page":"237","DOI":"10.1613\/jair.301","article-title":"Reinforcement learning: A survey","volume":"4","author":"LP Kaelbling","year":"1996","journal-title":"Journal of artificial intelligence research"},{"issue":"1","key":"ref31","doi-asserted-by":"crossref","first-page":"90","DOI":"10.1177\/027836498600500106","article-title":"Real-time obstacle avoidance for manipulators and mobile robots","volume":"5","author":"O Khatib","year":"1986","journal-title":"The international journal of robotics research"},{"issue":"4","key":"ref32","doi-asserted-by":"crossref","first-page":"92","DOI":"10.1177\/027836498900800406","article-title":"Motor schema-based mobile robot navigation","volume":"8","author":"RC Arkin","year":"1989","journal-title":"The International journal of robotics research"},{"issue":"4","key":"ref33","doi-asserted-by":"crossref","first-page":"288","DOI":"10.1016\/j.robot.2005.11.004","article-title":"Visual navigation and obstacle avoidance using a steering potential function","volume":"54","author":"WH Huang","year":"2006","journal-title":"Robotics and Autonomous Systems"},{"key":"ref34","doi-asserted-by":"crossref","unstructured":"Abbeel P, Ng AY. Apprenticeship learning via inverse reinforcement learning. In: Proceedings of the twenty-first international conference on Machine learning. ACM; 2004. p. 1.","DOI":"10.1145\/1015330.1015430"},{"key":"ref35","unstructured":"Ziebart BD, Maas A, Bagnell JA, Dey AK. Maximum entropy inverse reinforcement learning. In: Proceedings of the 23rd national conference on Artificial intelligence-Volume 3. AAAI Press; 2008. p. 1433\u20131438."},{"key":"ref36","unstructured":"Ramachandran D, Amir E. Bayesian inverse reinforcement learning. In: Proceedings of the 20th International Joint Conference on Artificial Intelligence. Morgan Kaufmann Publishers Inc.; 2007. p. 2586\u20132591."},{"key":"ref37","first-page":"31","article-title":"Machine Learning and Knowledge Discovery in Databases","author":"M Lopes","year":"2009"},{"key":"ref38","unstructured":"Babes M, Marivate V, Subramanian K, Littman ML. Apprenticeship learning about multiple intentions. In: Proceedings of the 28th International Conference on Machine Learning (ICML-11); 2011. p. 897\u2013904."},{"issue":"4","key":"ref39","doi-asserted-by":"crossref","first-page":"793","DOI":"10.1109\/TCYB.2014.2336867","article-title":"Hierarchical bayesian inverse reinforcement learning","volume":"45","author":"J Choi","year":"2015","journal-title":"IEEE transactions on cybernetics"},{"key":"ref40","unstructured":"Van Seijen H, Fatemi M, Romoff J, Laroche R, Barnes T, Tsang J. Hybrid reward architecture for reinforcement learning. In: Advances in Neural Information Processing Systems; 2017. p. 5392\u20135402."},{"key":"ref41","doi-asserted-by":"crossref","first-page":"399","DOI":"10.1613\/jair.1000","article-title":"Efficient solution algorithms for factored MDPs","author":"C Guestrin","year":"2003","journal-title":"Journal of Artificial Intelligence Research"},{"key":"ref42","doi-asserted-by":"crossref","unstructured":"Rohanimanesh K, Mahadevan S. Coarticulation: An approach for generating concurrent plans in Markov decision processes. In: Proceedings of the 22nd International Conference on Machine Learning. ACM; 2005. p. 720\u2013727.","DOI":"10.1145\/1102351.1102442"},{"key":"ref43","doi-asserted-by":"crossref","first-page":"227","DOI":"10.1613\/jair.639","article-title":"Hierarchical reinforcement learning with the MAXQ value function decomposition","volume":"13","author":"TG Dietterich","year":"2000","journal-title":"J Artif Intell Res(JAIR)"},{"issue":"1","key":"ref44","doi-asserted-by":"crossref","first-page":"181","DOI":"10.1016\/S0004-3702(99)00052-1","article-title":"Between MDPs and semi-MDPs: A framework for temporal abstraction in reinforcement learning","volume":"112","author":"RS Sutton","year":"1999","journal-title":"Artificial intelligence"},{"issue":"8","key":"ref45","doi-asserted-by":"crossref","first-page":"e1003779","DOI":"10.1371\/journal.pcbi.1003779","article-title":"Optimal behavioral hierarchy","volume":"10","author":"A Solway","year":"2014","journal-title":"PLoS computational biology"},{"issue":"6","key":"ref46","doi-asserted-by":"crossref","first-page":"1027","DOI":"10.1016\/j.conb.2012.06.001","article-title":"The root of all value: a neural common currency for choice","volume":"22","author":"DJ Levy","year":"2012","journal-title":"Current opinion in neurobiology"},{"issue":"7","key":"ref47","doi-asserted-by":"crossref","first-page":"e1005004","DOI":"10.1371\/journal.pcbi.1005004","article-title":"Properties of neurons in external globus pallidus can support optimal action selection","volume":"12","author":"R Bogacz","year":"2016","journal-title":"PLoS Comput Biol"},{"issue":"6","key":"ref48","doi-asserted-by":"crossref","first-page":"1830","DOI":"10.1093\/brain\/aww075","article-title":"The human subthalamic nucleus encodes the subjective value of reward and the cost of effort during decision-making","volume":"139","author":"A Z\u00e9non","year":"2016","journal-title":"Brain"},{"issue":"19","key":"ref49","doi-asserted-by":"crossref","first-page":"6234","DOI":"10.1523\/JNEUROSCI.5570-08.2009","article-title":"Adaptive gaze control in natural environments","volume":"29","author":"J Jovancevic-Misic","year":"2009","journal-title":"Journal of Neuroscience"},{"issue":"12","key":"ref50","doi-asserted-by":"crossref","first-page":"9","DOI":"10.1167\/6.12.9","article-title":"Control of attention and gaze in complex environments","volume":"6","author":"J Jovancevic","year":"2006","journal-title":"Journal of Vision"},{"issue":"1","key":"ref51","doi-asserted-by":"crossref","first-page":"90","DOI":"10.1086\/208899","article-title":"Adding asymmetrically dominated alternatives: Violations of regularity and the similarity hypothesis","volume":"9","author":"J Huber","year":"1982","journal-title":"Journal of consumer research"},{"key":"ref52","doi-asserted-by":"crossref","DOI":"10.7551\/mitpress\/9780262028615.001.0001","article-title":"Brain computation as hierarchical abstraction","author":"DH Ballard","year":"2015"},{"key":"ref53","doi-asserted-by":"crossref","unstructured":"Bengio Y, Louradour J, Collobert R, Weston J. Curriculum learning. In: Proceedings of the 26th annual international conference on machine learning. ACM; 2009. p. 41\u201348.","DOI":"10.1145\/1553374.1553380"},{"key":"ref54","unstructured":"Baker CL, Tenenbaum JB, Saxe RR. Goal inference as inverse planning. In: Proceedings of the Annual Meeting of the Cognitive Science Society. vol. 29; 2007."},{"issue":"7540","key":"ref55","doi-asserted-by":"crossref","first-page":"529","DOI":"10.1038\/nature14236","article-title":"Human-level control through deep reinforcement learning","volume":"518","author":"V Mnih","year":"2015","journal-title":"Nature"},{"key":"ref56","unstructured":"Bhat S, Isbell CL, Mateas M. On the difficulty of modular reinforcement learning for real-world partial programming. In: Proceedings of the National Conference on Artificial Intelligence. vol. 21. Menlo Park, CA; Cambridge, MA; London; AAAI Press; MIT Press; 1999; 2006. p. 318."},{"key":"ref57","unstructured":"Ring M, Schaul T. Q-error as a selection mechanism in modular reinforcement-learning systems. In: Proceedings of International Joint Conference on Artificial Intelligence. vol. 22; 2011. p. 1452."},{"key":"ref58","doi-asserted-by":"crossref","unstructured":"Zhang R, Song Z, Ballard DH. Global Policy Construction in Modular Reinforcement Learning. In: AAAI; 2015. p. 4226\u20134227.","DOI":"10.1609\/aaai.v29i1.9736"},{"issue":"1636","key":"ref59","doi-asserted-by":"crossref","first-page":"20130044","DOI":"10.1098\/rstb.2013.0044","article-title":"Predicting human visuomotor behaviour in a driving task","volume":"369","author":"L Johnson","year":"2014","journal-title":"Philosophical Transactions of the Royal Society of London B: Biological Sciences"},{"issue":"46","key":"ref60","doi-asserted-by":"crossref","first-page":"15497","DOI":"10.1523\/JNEUROSCI.3270-14.2014","article-title":"Attention, reward, and information seeking","volume":"34","author":"J Gottlieb","year":"2014","journal-title":"Journal of Neuroscience"},{"issue":"8","key":"ref61","doi-asserted-by":"crossref","first-page":"1224","DOI":"10.1016\/j.cub.2018.03.008","article-title":"Gaze and the control of foot placement when walking in natural terrain","volume":"28","author":"JS Matthis","year":"2018","journal-title":"Current Biology"}],"updated-by":[{"DOI":"10.1371\/journal.pcbi.1006518","type":"new_version","label":"New version","source":"publisher","updated":{"date-parts":[[2018,11,6]],"date-time":"2018-11-06T00:00:00Z","timestamp":1541462400000}}],"container-title":["PLOS Computational Biology"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/dx.plos.org\/10.1371\/journal.pcbi.1006518","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,9,4]],"date-time":"2022-09-04T13:26:49Z","timestamp":1662298009000},"score":1,"resource":{"primary":{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pcbi.1006518"}},"subtitle":[],"editor":[{"given":"Samuel J.","family":"Gershman","sequence":"first","affiliation":[],"role":[{"role":"editor","vocabulary":"crossref"}]}],"short-title":[],"issued":{"date-parts":[[2018,10,25]]},"references-count":61,"journal-issue":{"issue":"10","published-online":{"date-parts":[[2018,10,25]]}},"URL":"https:\/\/doi.org\/10.1371\/journal.pcbi.1006518","relation":{"has-preprint":[{"id-type":"doi","id":"10.1101\/412155","asserted-by":"object"}]},"ISSN":["1553-7358"],"issn-type":[{"value":"1553-7358","type":"electronic"}],"subject":[],"published":{"date-parts":[[2018,10,25]]}}}