{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,28]],"date-time":"2026-08-28T08:14:31Z","timestamp":1787904871975,"version":"build-2784847793"},"reference-count":51,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"2","license":[{"start":{"date-parts":[[2020,6,1]],"date-time":"2020-06-01T00:00:00Z","timestamp":1590969600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2020,6,1]],"date-time":"2020-06-01T00:00:00Z","timestamp":1590969600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2020,6,1]],"date-time":"2020-06-01T00:00:00Z","timestamp":1590969600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100003176","name":"Ministerio de Educaci\u00f3n, Cultura y Deporte","doi-asserted-by":"publisher","award":["PRX17\/00467"],"award-info":[{"award-number":["PRX17\/00467"]}],"id":[{"id":"10.13039\/501100003176","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Future of Life Institute","award":["RFP2-152"],"award-info":[{"award-number":["RFP2-152"]}]},{"DOI":"10.13039\/501100013410","name":"INCIBE","doi-asserted-by":"crossref","award":["Ayud Inv. Ciberseguridad"],"award-info":[{"award-number":["Ayud Inv. Ciberseguridad"]}],"id":[{"id":"10.13039\/501100013410","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100003329","name":"Ministerio de Economia y Competitividad","doi-asserted-by":"crossref","award":["TIN 2015-69175-C4-1-R"],"award-info":[{"award-number":["TIN 2015-69175-C4-1-R"]}],"id":[{"id":"10.13039\/501100003329","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000181","name":"Air Force Office of Scientific Research","doi-asserted-by":"crossref","award":["FA9550-17-1-0287"],"award-info":[{"award-number":["FA9550-17-1-0287"]}],"id":[{"id":"10.13039\/100000181","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100003359","name":"Generalitat Valenciana","doi-asserted-by":"crossref","award":["BEST\/2017\/045"],"award-info":[{"award-number":["BEST\/2017\/045"]}],"id":[{"id":"10.13039\/501100003359","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100003359","name":"Generalitat Valenciana","doi-asserted-by":"crossref","award":["PROMETEOII\/2015\/013"],"award-info":[{"award-number":["PROMETEOII\/2015\/013"]}],"id":[{"id":"10.13039\/501100003359","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Games"],"published-print":{"date-parts":[[2020,6]]},"DOI":"10.1109\/tg.2018.2883773","type":"journal-article","created":{"date-parts":[[2018,11,28]],"date-time":"2018-11-28T15:17:01Z","timestamp":1543418221000},"page":"121-131","source":"Crossref","is-referenced-by-count":16,"title":["Dual Indicators to Analyze AI Benchmarks: Difficulty, Discrimination, Ability, and Generality"],"prefix":"10.1109","volume":"12","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2902-6477","authenticated-orcid":false,"given":"Fernando","family":"Martinez-Plumed","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9746-7632","authenticated-orcid":false,"given":"Jose","family":"Hernandez-Orallo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref39","article-title":"Learning to play in a day: Faster deep reinforcement learning by optimality tightening","author":"he","year":"2016"},{"key":"ref38","first-page":"2094","article-title":"Deep reinforcement learning with double q-learning","author":"van hasselt","year":"0","journal-title":"Proc 30th AAAI Conf Artif Intell"},{"key":"ref33","first-page":"1610","article-title":"Classical planning with simulators: Results on the atari video games","author":"lipovetzky","year":"0","journal-title":"Proc Int Joint Conf Artif Intell"},{"key":"ref32","article-title":"Game-independent ai agents for playing Atari 2600 console games","author":"naddaf","year":"2010"},{"key":"ref31","first-page":"122","article-title":"Matching games and algorithms for general video game playing","author":"bontrager","year":"0","journal-title":"Proc 12th AI Interact Digit Entertainment Conf"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/CIG.2015.7317941"},{"key":"ref37","article-title":"Prioritized experience replay","author":"schaul","year":"0","journal-title":"Proc Int Conf Learn Represent"},{"key":"ref36","article-title":"Dueling network architectures for deep reinforcement learning","author":"wang","year":"0","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref35","article-title":"Learning and generalization in Atari games","author":"furelos-blanco","year":"2015"},{"key":"ref34","article-title":"Blind search for Atari-like online planning revisited","author":"shleyfman","year":"0","journal-title":"Heurist and Search for Domain-Indep Planning"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/CIG.2017.8080410"},{"key":"ref27","article-title":"What the no free lunch theorems really mean; how to improve search algorithms","author":"wolpert","year":"2012"},{"key":"ref29","article-title":"Analysing results from ai benchmarks: Key indicators and how to obtain them","author":"mart\u00ednez-plumed","year":"2018"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1016\/S0004-3702(01)00129-1"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1147\/rd.33.0210"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-54157-0_25"},{"key":"ref22","first-page":"1","article-title":"Ai results for the Atari 2600 games: difficulty and discrimination using IRT","author":"mart\u00ednez-plumed","year":"0","journal-title":"Proc 2nd Int Workshop Eval Gen -Purpose AI"},{"key":"ref21","author":"embretson","year":"2000","journal-title":"Item Response Theory for Psychologists"},{"key":"ref24","first-page":"1140","article-title":"Making sense of item response theory in machine learning","author":"mart\u00ednez-plumed","year":"0","journal-title":"Proc Eur Conf Artif Intell"},{"key":"ref23","first-page":"1","article-title":"Analysis of instance hardness in machine learning using item response theory","author":"prud\u00eancio","year":"0","journal-title":"Proc 2nd Int Workshop Learn Over Multiple Contexts Porto Portugal"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/4235.585893"},{"key":"ref25","article-title":"Some latent trait models and their use in inferring an examinees ability","author":"birnbaum","year":"1968","journal-title":"Statistical Theories of Mental Test Scores"},{"key":"ref50","article-title":"Re-evaluating evaluation","author":"balduzzi","year":"2018"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-32323-2"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/TCIAIG.2014.2339221"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1017\/9781316594179"},{"key":"ref40","article-title":"Neural episodic control","author":"pritzel","year":"2017"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v38i3.2748"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2010.09.006"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1038\/540323a"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1613\/jair.3912"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/TCIAIG.2015.2402393"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v34i3.2492"},{"key":"ref18","first-page":"4326","article-title":"Aibirds: The angry birds artificial intelligence competition","author":"renz","year":"0","journal-title":"Proc 29th AAAI Conf Artif Intell"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/TCIAIG.2013.2286295"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1038\/nature16961"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2012.06.009"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v36i3.2571"},{"key":"ref5","article-title":"UCI machine learning repository","author":"lichman","year":"2013"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1007\/s10462-016-9505-7"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"key":"ref49","first-page":"540","article-title":"Width and serialization of classical planning problems","author":"lipovetzky","year":"0","journal-title":"Proc 20th Eur Conf Artif Intell"},{"key":"ref9","doi-asserted-by":"crossref","first-page":"529","DOI":"10.1038\/nature14236","article-title":"Human-level control through deep reinforcement learning","volume":"518","author":"mnih","year":"2015","journal-title":"Nature"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/TCIAIG.2012.2186810"},{"key":"ref45","article-title":"General game playing: Overview of the AAAI competition","volume":"26","author":"genesereth","year":"2005","journal-title":"AI Mag"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1080\/15366367.2013.831680"},{"key":"ref47","first-page":"351","article-title":"Rolling horizon evolution versus tree search for navigation in single-player real-time games","author":"perez-liebana","year":"0","journal-title":"Proc 15th Genetic Evol Comput Conf"},{"key":"ref42","article-title":"The Reactor: A fast and sample-efficient actor-critic agent for reinforcement learning","author":"gruslys","year":"0","journal-title":"Proc Int Conf on Learning Rep"},{"key":"ref41","first-page":"43","article-title":"Pairwise relative offset features for atari 2600 games","author":"talvitie","year":"0","journal-title":"Proc AAAI Workshop Learn General Competency Video Games"},{"key":"ref44","article-title":"Evolution strategies as a scalable alternative to reinforcement learning","author":"salimans","year":"2017"},{"key":"ref43","article-title":"Massively parallel methods for deep reinforcement learning","author":"nair","year":"2015"}],"container-title":["IEEE Transactions on Games"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/7782673\/9119230\/08550672.pdf?arnumber=8550672","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,4,27]],"date-time":"2022-04-27T08:55:57Z","timestamp":1651049757000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/8550672\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,6]]},"references-count":51,"journal-issue":{"issue":"2"},"URL":"https:\/\/doi.org\/10.1109\/tg.2018.2883773","relation":{},"ISSN":["2475-1502","2475-1510"],"issn-type":[{"value":"2475-1502","type":"print"},{"value":"2475-1510","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,6]]}}}