{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,25]],"date-time":"2026-02-25T18:49:00Z","timestamp":1772045340596,"version":"3.50.1"},"publisher-location":"Berlin, Heidelberg","reference-count":43,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"value":"9783642364051","type":"print"},{"value":"9783642364068","type":"electronic"}],"license":[{"start":{"date-parts":[[2013,1,1]],"date-time":"2013-01-01T00:00:00Z","timestamp":1356998400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2013,1,1]],"date-time":"2013-01-01T00:00:00Z","timestamp":1356998400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2013]]},"DOI":"10.1007\/978-3-642-36406-8_4","type":"book-chapter","created":{"date-parts":[[2013,1,31]],"date-time":"2013-01-31T11:07:18Z","timestamp":1359630438000},"page":"101-128","source":"Crossref","is-referenced-by-count":3,"title":["Counter-Factual Reinforcement Learning: How to Model Decision-Makers That Anticipate the Future"],"prefix":"10.1007","author":[{"given":"Ritchie","family":"Lee","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"David H.","family":"Wolpert","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"James","family":"Bono","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Scott","family":"Backhaus","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Russell","family":"Bent","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Brendan","family":"Tracey","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","reference":[{"key":"4_CR1","doi-asserted-by":"crossref","unstructured":"Bono, J., Wolpert, D.H.: Decision-theoretic prediction and policy design of gdp slot auctions (2011), Available at SSRN: \n                    http:\/\/ssrn.com\/abstract=1815222","DOI":"10.2139\/ssrn.1815222"},{"key":"4_CR2","doi-asserted-by":"crossref","unstructured":"Brunner, C., Camerer, C.F., Goeree, J.K.: A correction and re-examination of \u2019stationary concepts for experimental 2x2 games\u2019. American Economic Review (2010)","DOI":"10.1257\/aer.101.2.1029"},{"key":"4_CR3","unstructured":"Busoniu, L., Babuska, R., De Schutter, B., Damien, E.: Reinforcement learning and dynamic programming using function approximators. CRC Press (2010)"},{"issue":"1","key":"4_CR4","doi-asserted-by":"publisher","first-page":"61","DOI":"10.1007\/BF00055711","volume":"2","author":"C.F. Camerer","year":"1989","unstructured":"Camerer, C.F.: An experimental test of several generalized utility theories. Journal of Risk and Uncertainty\u00a02(1), 61\u2013104 (1989)","journal-title":"Journal of Risk and Uncertainty"},{"key":"4_CR5","unstructured":"Camerer, C.F.: Behavioral game theory: experiments in strategic interaction. Princeton University Press (2003)"},{"issue":"3","key":"4_CR6","doi-asserted-by":"publisher","first-page":"861","DOI":"10.1162\/0033553041502225","volume":"119","author":"C. Camerer","year":"2006","unstructured":"Camerer, C., Ho, T.H., Chong, J.K.: A cognitive hierarchy model of games. Quarterly Journal of Economics\u00a0119(3), 861\u2013898 (2006)","journal-title":"Quarterly Journal of Economics"},{"key":"4_CR7","unstructured":"C\u00e1rdenas, A., Amin, A., Sastry, S.: Research challenges for the security of control systems. In: Proceedings of the 3rd Conference on Hot Topics in Security, Berkeley, CA, USA, pp. 6:1\u20136:6. USENIX Association (2008)"},{"issue":"4","key":"4_CR8","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1109\/4235.942536","volume":"5","author":"K. Chellapilla","year":"2001","unstructured":"Chellapilla, K., Fogel, D.B.: Evolving an expert checkers playing program without using human expertise. IEEE Transactions on Evolutionary Computation\u00a05(4), 422\u2013428 (2001)","journal-title":"IEEE Transactions on Evolutionary Computation"},{"issue":"5","key":"4_CR9","doi-asserted-by":"publisher","first-page":"1737","DOI":"10.1257\/aer.96.5.1737","volume":"96","author":"M. Costa-Gomes","year":"2006","unstructured":"Costa-Gomes, M., Crawford, V.: Cognition and behavior in two-person guessing games: An experimental study. American Economic Review\u00a096(5), 1737\u20131768 (2006)","journal-title":"American Economic Review"},{"key":"4_CR10","doi-asserted-by":"crossref","unstructured":"Costa-Gomes, M.A., Crawford, V.P., Iriberri, N.: Comparing models of strategic thinking in Van Huyck, Battalio, and Beil\u2019s coordination games. Journal of the European Economic Association (2009)","DOI":"10.1162\/JEEA.2009.7.2-3.365"},{"key":"4_CR11","unstructured":"Crawford, V.P.: Level-k thinking. Plenary lecture. 2007 North American Meeting of the Economic Science Association. Tucson, Arizona (2007)"},{"issue":"2","key":"4_CR12","doi-asserted-by":"publisher","first-page":"285","DOI":"10.1016\/S0896-6273(02)00963-7","volume":"36","author":"P. Dayan","year":"2002","unstructured":"Dayan, P., Balleine, B.W.: Reward, motivation, and reinforcement learning. Neuron\u00a036(2), 285\u2013298 (2002)","journal-title":"Neuron"},{"key":"4_CR13","doi-asserted-by":"crossref","unstructured":"Fogel, D.B.: Evolutionary computation: Toward a new philosophy of machine intelligence, 3rd edn. IEEE Press (2006)","DOI":"10.1002\/0471749214"},{"key":"4_CR14","unstructured":"Fudenberg, D., Levine, D.K.: The theory of learning in games. MIT Press (1998)"},{"key":"4_CR15","doi-asserted-by":"crossref","first-page":"49","DOI":"10.1613\/jair.1579","volume":"24","author":"P.J. Gmytrasiewicz","year":"2005","unstructured":"Gmytrasiewicz, P.J., Doshi, P.: A framework for sequential planning in multi-agent settings. Journal of Artificial Intelligence Research\u00a024, 49\u201379 (2005)","journal-title":"Journal of Artificial Intelligence Research"},{"key":"4_CR16","doi-asserted-by":"crossref","unstructured":"Halpern, J.Y., Rego, L.C.: Extensive games with possibly unaware players (2007) (Working paper), \n                    http:\/\/www.cs.cornell.edu\/home\/halpern\/papers\/aamas06.pdf","DOI":"10.1145\/1160633.1160765"},{"key":"4_CR17","doi-asserted-by":"crossref","unstructured":"Harsanyi, J.: Games with Incomplete Information Played by Bayesian Players, I-III. Part I. The Basic Model. Management Science\u00a014(3) (1967)","DOI":"10.1287\/mnsc.14.3.159"},{"key":"4_CR18","doi-asserted-by":"crossref","first-page":"237","DOI":"10.1613\/jair.301","volume":"4","author":"L.P. Kaelbling","year":"1996","unstructured":"Kaelbling, L.P., Littman, M.L., Moore, A.W.: Reinforcement learning: A survey. Journal of Artificial Intelligence Research\u00a04, 237\u2013285 (1996)","journal-title":"Journal of Artificial Intelligence Research"},{"key":"4_CR19","doi-asserted-by":"crossref","unstructured":"Kagel, J.H., Roth, A.E.: The handbook of experimental economics. Princeton University Press (1995)","DOI":"10.1515\/9780691213255"},{"issue":"1","key":"4_CR20","doi-asserted-by":"publisher","first-page":"29","DOI":"10.2307\/2951777","volume":"61","author":"M. Kandori","year":"1993","unstructured":"Kandori, M., Mailath, M., Rob, R.: Learning, mutation, and long run equilibria in games. Econometrica\u00a061(1), 29\u201353 (1993)","journal-title":"Econometrica"},{"key":"4_CR21","unstructured":"Koller, D., Friedman, N.: Probabilistic graphical models: principles and techniques. MIT Press (2009)"},{"key":"4_CR22","volume-title":"Information theory and statistics","author":"S. Kullback","year":"1959","unstructured":"Kullback, S.: Information theory and statistics. John Wiley and Sons, New York (1959)"},{"key":"4_CR23","volume-title":"Power system stability and control","author":"P. Kundur","year":"1993","unstructured":"Kundur, P.: Power system stability and control. McGraw-Hill, New York (1993)"},{"key":"4_CR24","series-title":"ISRL","doi-asserted-by":"publisher","first-page":"75","DOI":"10.1007\/978-3-642-24647-0_4","volume-title":"Decision Making with Imperfect Decision Makers","author":"R. Lee","year":"2012","unstructured":"Lee, R., Wolpert, D.: Game Theoretic Modeling of Pilot Behavior during Mid-Air Encounters. In: Guy, T.V., K\u00e1rn\u00fd, M., Wolpert, D.H. (eds.) Decision Making with Imperfect Decision Makers. ISRL, vol.\u00a028, pp. 75\u2013111. Springer, Heidelberg (2012)"},{"issue":"4","key":"4_CR25","doi-asserted-by":"publisher","first-page":"343","DOI":"10.3758\/CABN.9.4.343","volume":"9","author":"T. Maia","year":"2009","unstructured":"Maia, T.: Reinforcement learning, conditioning, and the brain: Successes and challenges. Cognitive, Affective, & Behavioral Neuroscience\u00a09(4), 343\u2013364 (2009)","journal-title":"Cognitive, Affective, & Behavioral Neuroscience"},{"key":"4_CR26","doi-asserted-by":"publisher","first-page":"154","DOI":"10.1038\/nn.2723","volume":"14","author":"T.V. Maia","year":"2011","unstructured":"Maia, T.V., Frank, M.J.: From reinforcement learning models to psychiatric and neurological. Nature Neuroscience\u00a014, 154\u2013162 (2011)","journal-title":"Nature Neuroscience"},{"issue":"1","key":"4_CR27","doi-asserted-by":"publisher","first-page":"6","DOI":"10.1006\/game.1995.1023","volume":"10","author":"R. McKelvey","year":"1995","unstructured":"McKelvey, R., Palfrey, T.R.: Quantal response equilibria for normal form games. Games and Economic Behavior\u00a010(1), 6\u201338 (1995)","journal-title":"Games and Economic Behavior"},{"key":"4_CR28","doi-asserted-by":"crossref","first-page":"9","DOI":"10.1023\/A:1009905800005","volume":"1","author":"R. McKelvey","year":"1998","unstructured":"McKelvey, R., Palfrey, T.R.: Quantal response equilibria for extensive form games. Experimental Economics\u00a01, 9\u201341 (1998), 10.1023\/A:1009905800005","journal-title":"Experimental Economics"},{"key":"4_CR29","doi-asserted-by":"crossref","first-page":"241","DOI":"10.1613\/jair.613","volume":"11","author":"D.E. Moriarty","year":"1999","unstructured":"Moriarty, D.E., Schultz, A.C., Grefenstette, J.J.: Evolutionary algorithms for reinforcement learning. The Journal of Artificial Intelligence Research\u00a011, 241\u2013276 (1999)","journal-title":"The Journal of Artificial Intelligence Research"},{"key":"4_CR30","unstructured":"Myerson, R.B.: Game theory: Analysis of conflict. Harvard University Press (1997)"},{"issue":"5","key":"4_CR31","first-page":"1313","volume":"85","author":"R. Nagel","year":"1995","unstructured":"Nagel, R.: Unraveling in guessing games: An experimental study. The American Economic Review\u00a085(5), 1313\u20131326 (1995)","journal-title":"The American Economic Review"},{"key":"4_CR32","volume-title":"The handbook of experimental economics","author":"C.R. Plott","year":"2008","unstructured":"Plott, C.R., Smith, V.L.: The handbook of experimental economics. North-Holland, Oxford (2008)"},{"key":"4_CR33","doi-asserted-by":"crossref","unstructured":"Robert, C.P., Casella, G.: Monte Carlo statistical methods, 2nd edn. Springer (2004)","DOI":"10.1007\/978-1-4757-4145-2"},{"key":"4_CR34","unstructured":"Rummery, G.A., Niranja, M.: Online Q-learning using connectionist systems. Technical report CUED\/F-INFENG\/TR 166. Engineering department, Cambridge University (1994)"},{"issue":"2","key":"4_CR35","doi-asserted-by":"publisher","first-page":"129","DOI":"10.1037\/h0042769","volume":"63","author":"H.A. Simon","year":"1956","unstructured":"Simon, H.A.: Rational choice and the structure of the environment. Psychological Review\u00a063(2), 129\u2013138 (1956)","journal-title":"Psychological Review"},{"key":"4_CR36","doi-asserted-by":"crossref","unstructured":"Singh, S.P., Jaakkola, T., Jordan, M.I.: Learning without state-estimation in partially observable Markovian decision problems. In: Proceedings of the Eleventh International Conference on Machine Learning, San Francisco, pp. 284\u2013292 (1994)","DOI":"10.1016\/B978-1-55860-335-6.50042-8"},{"issue":"1","key":"4_CR37","doi-asserted-by":"publisher","first-page":"218","DOI":"10.1006\/game.1995.1031","volume":"10","author":"D.O. Stahl","year":"1995","unstructured":"Stahl, D.O., Wilson, P.W.: On players\u2019 models of other players: Theory and experimental evidence. Games and Economic Behavior\u00a010(1), 218\u2013254 (1995)","journal-title":"Games and Economic Behavior"},{"key":"4_CR38","doi-asserted-by":"crossref","unstructured":"Sutton, R.S., Barto, A.G.: Reinforcement learning: An introduction. MIT Press (1998)","DOI":"10.1109\/TNN.1998.712192"},{"issue":"5","key":"4_CR39","doi-asserted-by":"publisher","first-page":"965","DOI":"10.1109\/JPROC.2005.847249","volume":"93","author":"K. Tomsovic","year":"2005","unstructured":"Tomsovic, K., Bakken, D.E., Venkatasubramanian, V., Bose, A.: Designing the next generation of real-time control, communication, and computations for large power systems. Proceedings of the IEEE\u00a093(5), 965\u2013979 (2005)","journal-title":"Proceedings of the IEEE"},{"issue":"6","key":"4_CR40","doi-asserted-by":"publisher","first-page":"1063","DOI":"10.1109\/JPROC.2011.2116750","volume":"99","author":"K. Turitsyn","year":"2011","unstructured":"Turitsyn, K., Sulc, P., Backhaus, S., Chertkov, M.: Options for control of reactive power by distributed photovoltaic generators. Proceedings of the IEEE\u00a099(6), 1063\u20131073 (2011)","journal-title":"Proceedings of the IEEE"},{"key":"4_CR41","doi-asserted-by":"crossref","unstructured":"Wolpert, D.H., Bono, J.W.: Distribution-valued solution concepts. Working paper (2011)","DOI":"10.2139\/ssrn.1622463"},{"key":"4_CR42","unstructured":"Wolpert, D.H.: Unawareness, information theory, and multiagent influence diagrams. Working paper (2012)"},{"key":"4_CR43","doi-asserted-by":"crossref","unstructured":"Wright, J.R., Leyton-Brown, K.: Beyond equilibrium: Predicting human behavior in normal form games. In: Twenty-Fourth Conference on Artificial Intelligence, AAAI 2010 (2010)","DOI":"10.1145\/1807406.1807449"}],"container-title":["Studies in Computational Intelligence","Decision Making and Imperfection"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-642-36406-8_4","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,2,19]],"date-time":"2023-02-19T09:16:28Z","timestamp":1676798188000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-642-36406-8_4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013]]},"ISBN":["9783642364051","9783642364068"],"references-count":43,"URL":"https:\/\/doi.org\/10.1007\/978-3-642-36406-8_4","relation":{},"ISSN":["1860-949X","1860-9503"],"issn-type":[{"value":"1860-949X","type":"print"},{"value":"1860-9503","type":"electronic"}],"subject":[],"published":{"date-parts":[[2013]]}}}