{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,29]],"date-time":"2026-09-29T13:29:16Z","timestamp":1790688556786,"version":"4.1.0"},"reference-count":253,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2016,8,19]],"date-time":"2016-08-19T00:00:00Z","timestamp":1471564800000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"name":"MINECO","award":["2013-45732-C4-1-P"],"award-info":[{"award-number":["2013-45732-C4-1-P"]}]},{"name":"MINECO","award":["TIN 2015-69175-C4-1-R"],"award-info":[{"award-number":["TIN 2015-69175-C4-1-R"]}]},{"DOI":"10.13039\/501100003359","name":"Generalitat Valenciana","doi-asserted-by":"publisher","award":["PROMETEOII2015\/013"],"award-info":[{"award-number":["PROMETEOII2015\/013"]}],"id":[{"id":"10.13039\/501100003359","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Artif Intell Rev"],"published-print":{"date-parts":[[2017,10]]},"DOI":"10.1007\/s10462-016-9505-7","type":"journal-article","created":{"date-parts":[[2016,8,19]],"date-time":"2016-08-19T14:15:24Z","timestamp":1471616124000},"page":"397-447","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":138,"title":["Evaluation in artificial intelligence: from task-oriented to ability-oriented measurement"],"prefix":"10.1007","volume":"48","author":[{"given":"Jos\u00e9","family":"Hern\u00e1ndez-Orallo","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2016,8,19]]},"reference":[{"key":"9505_CR1","unstructured":"Abel D, Agarwal A, Diaz F, Krishnamurthy A, Schapire RE (2016) Exploratory gradient boosting for reinforcement learning in complex domains. arXiv preprint arXiv:1603.04119"},{"issue":"1","key":"9505_CR2","doi-asserted-by":"crossref","first-page":"25","DOI":"10.1609\/aimag.v33i1.2322","volume":"33","author":"S Adams","year":"2012","unstructured":"Adams S, Arel I, Bach J, Coop R, Furlan R, Goertzel B, Hall JS, Samsonovich A, Scheutz M, Schlesinger M, Shapiro SC, Sowa J (2012) Mapping the landscape of human-level artificial general intelligence. AI Mag 33(1):25\u201342","journal-title":"AI Mag"},{"issue":"1","key":"9505_CR3","doi-asserted-by":"crossref","first-page":"78","DOI":"10.1609\/aimag.v37i1.2643","volume":"37","author":"SS Adams","year":"2016","unstructured":"Adams SS, Banavar G, Campbell M (2016) I-athlon: towards a multi-dimensional Turing test. AI Mag 37(1):78\u201384","journal-title":"AI Mag"},{"key":"9505_CR4","first-page":"255","volume":"17","author":"J Alcal\u00e1","year":"2010","unstructured":"Alcal\u00e1 J, Fern\u00e1ndez A, Luengo J, Derrac J, Garc\u00eda S, S\u00e1nchez L, Herrera F (2010) Keel data-mining software tool: data set repository, integration of algorithms and experimental analysis framework. J Mult Valued Logic Soft Comput 17:255\u2013287","journal-title":"J Mult Valued Logic Soft Comput"},{"issue":"5","key":"9505_CR5","doi-asserted-by":"crossref","first-page":"815","DOI":"10.1016\/S0191-8869(97)00054-8","volume":"23","author":"JRM Alexander","year":"1997","unstructured":"Alexander JRM, Smales S (1997) Intelligence, learning and long-term memory. Personal Individ Differ 23(5):815\u2013825","journal-title":"Personal Individ Differ"},{"key":"9505_CR6","doi-asserted-by":"crossref","unstructured":"Alpcan T, Everitt T, Hutter M (2014) Can we measure the difficulty of an optimization problem? In: IEEE information theory workshop (ITW)","DOI":"10.1109\/ITW.2014.6970853"},{"key":"9505_CR7","doi-asserted-by":"crossref","unstructured":"Alur R, Bodik R, Juniwal G, Martin MMK, Raghothaman M, Seshia SA, Singh R, Solar-Lezama A, Torlak E, Udupa A (2013) Syntax-guided synthesis. In: Formal methods in computer-aided design (FMCAD), 2013, IEEE, pp 1\u201317","DOI":"10.1109\/FMCAD.2013.6679385"},{"key":"9505_CR8","doi-asserted-by":"crossref","unstructured":"Alvarado N, Adams SS, Burbeck S, Latta C (2002) Beyond the Turing test: performance metrics for evaluating a computer simulation of the human mind. In: Proceedings of the 2nd international conference on development and learning, IEEE, pp 147\u2013152","DOI":"10.1109\/DEVLRN.2002.1011826"},{"issue":"3","key":"9505_CR9","doi-asserted-by":"crossref","first-page":"53","DOI":"10.1109\/MRA.2015.2448871","volume":"22","author":"F Amigoni","year":"2015","unstructured":"Amigoni F, Bastianelli E, Berghofer J, Bonarini A, Fontana G, Hochgeschwender N, Iocchi L, Kraetzschmar G, Lima P, Matteucci M, Miraldo P, Nardi D, Schiaffonati V (2015) Competitions for benchmarking: task and functionality scoring complete performance assessment. IEEE Robot Autom Mag 22(3):53\u201361","journal-title":"IEEE Robot Autom Mag"},{"issue":"5","key":"9505_CR10","doi-asserted-by":"crossref","first-page":"587","DOI":"10.1017\/S0140525X0300013X","volume":"26","author":"J Anderson","year":"2003","unstructured":"Anderson J, Lebiere C (2003) The Newell test for a theory of cognition. Behav Brain Sci 26(5):587\u2013601","journal-title":"Behav Brain Sci"},{"issue":"01","key":"9505_CR11","doi-asserted-by":"crossref","first-page":"11","DOI":"10.1017\/S0269888910000354","volume":"26","author":"J Anderson","year":"2011","unstructured":"Anderson J, Baltes J, Cheng CT (2011) Robotics competitions as benchmarks for AI research. Knowl Eng Rev 26(01):11\u201317","journal-title":"Knowl Eng Rev"},{"issue":"4","key":"9505_CR12","doi-asserted-by":"crossref","first-page":"13","DOI":"10.1109\/MCI.2010.938364","volume":"5","author":"I Arel","year":"2010","unstructured":"Arel I, Rose DC, Karnowski TP (2010) Deep machine learning\u2014a new frontier in artificial intelligence research. IEEE Comput Intell Mag 5(4):13\u201318","journal-title":"IEEE Comput Intell Mag"},{"issue":"1","key":"9505_CR13","doi-asserted-by":"crossref","first-page":"12","DOI":"10.1109\/TAMD.2009.2021702","volume":"1","author":"M Asada","year":"2009","unstructured":"Asada M, Hosoda K, Kuniyoshi Y, Ishiguro H, Inui T, Yoshikawa Y, Ogino M, Yoshida C (2009) Cognitive developmental robotics: a survey. IEEE Trans Auton Ment Dev 1(1):12\u201334","journal-title":"IEEE Trans Auton Ment Dev"},{"key":"9505_CR14","doi-asserted-by":"crossref","first-page":"493","DOI":"10.1613\/jair.4856","volume":"54","author":"H Aziz","year":"2015","unstructured":"Aziz H, Brill M, Fischer F, Harrenstein P, Lang J, Seedig HG (2015) Possible and necessary winners of partial tournaments. J Artif Intell Res 54:493\u2013534","journal-title":"J Artif Intell Res"},{"key":"9505_CR15","unstructured":"Bache K, Lichman M (2013) UCI machine learning repository. http:\/\/archive.ics.uci.edu\/ml"},{"key":"9505_CR16","doi-asserted-by":"crossref","unstructured":"Bagnall AJ, Zatuchna ZV (2005) On the classification of maze problems. In: Bull L, Kovacs T (eds) Foundations of learning classifier system. Studies in fuzziness and soft computing, vol. 183, Springer, pp 305\u2013316. http:\/\/rd.springer.com\/chapter\/10.1007\/11319122_12","DOI":"10.1007\/11319122_12"},{"issue":"5","key":"9505_CR17","doi-asserted-by":"crossref","first-page":"852","DOI":"10.1109\/21.376497","volume":"25","author":"D Baldwin","year":"1995","unstructured":"Baldwin D, Yadav SB (1995) The process of research investigations in artificial intelligence - a unified view. IEEE Trans Syst Man Cybern 25(5):852\u2013861","journal-title":"IEEE Trans Syst Man Cybern"},{"key":"9505_CR18","doi-asserted-by":"crossref","first-page":"253","DOI":"10.1613\/jair.3912","volume":"47","author":"MG Bellemare","year":"2013","unstructured":"Bellemare MG, Naddaf Y, Veness J, Bowling M (2013) The arcade learning environment: an evaluation platform for general agents. J Artif Intell Res 47:253\u2013279","journal-title":"J Artif Intell Res"},{"key":"9505_CR19","doi-asserted-by":"crossref","unstructured":"Besold TR (2014) A note on chances and limitations of psychometric ai. In: KI 2014: advances in artificial intelligence. Springer, pp 49\u201354","DOI":"10.1007\/978-3-319-11206-0_5"},{"issue":"2829, 10 Septem","key":"9505_CR20","doi-asserted-by":"crossref","first-page":"42","DOI":"10.1016\/S0262-4079(11)62227-6","volume":"211","author":"C Biever","year":"2011","unstructured":"Biever C (2011) Ultimate IQ: one test to rule them all. New Sci 211(2829, 10 September 2011):42\u201345","journal-title":"New Sci"},{"key":"9505_CR21","doi-asserted-by":"crossref","unstructured":"Borg M, Johansen SS, Thomsen DL, Kraus M (2012) Practical implementation of a graphics Turing test. In: Advances in visual computing. Springer, pp 305\u2013313","DOI":"10.1007\/978-3-642-33191-6_30"},{"key":"9505_CR22","unstructured":"Boring EG (1923) Intelligence as the tests test it. New Repub 35\u201337"},{"key":"9505_CR23","volume-title":"Superintelligence: paths, dangers, strategies","author":"N Bostrom","year":"2014","unstructured":"Bostrom N (2014) Superintelligence: paths, dangers, strategies. Oxford University Press, Oxford"},{"key":"9505_CR24","volume-title":"Metalearning: applications to data mining","author":"P Brazdil","year":"2008","unstructured":"Brazdil P, Carrier CG, Soares C, Vilalta R (2008) Metalearning: applications to data mining. Springer, New York"},{"issue":"3","key":"9505_CR25","doi-asserted-by":"crossref","first-page":"271","DOI":"10.1080\/0952813X.2010.502314","volume":"23","author":"S Bringsjord","year":"2011","unstructured":"Bringsjord S (2011) Psychometric artificial intelligence. J Exp Theor Artif Intell 23(3):271\u2013277","journal-title":"J Exp Theor Artif Intell"},{"key":"9505_CR26","unstructured":"Bringsjord S, Schimanski B (2003) What is artificial intelligence? Psychometric AI as an answer. In: International joint conference on artificial intelligence, pp 887\u2013893"},{"key":"9505_CR27","unstructured":"Brundage M (2016) Modeling progress in ai. AAAI 2016 Workshop on AI, Ethics, and Society"},{"key":"9505_CR28","volume-title":"Artificial intelligence as an experimental science","author":"BG Buchanan","year":"1988","unstructured":"Buchanan BG (1988) Artificial intelligence as an experimental science. Springer, New York"},{"issue":"1","key":"9505_CR29","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1177\/1745691610393980","volume":"6","author":"M Buhrmester","year":"2011","unstructured":"Buhrmester M, Kwang T, Gosling SD (2011) Amazon\u2019s mechanical turk a new source of inexpensive, yet high-quality, data? Perspect Psychol Sci 6(1):3\u20135","journal-title":"Perspect Psychol Sci"},{"key":"9505_CR30","unstructured":"Bursztein E, Aigrain J, Moscicki A, Mitchell JC (2014) The end is nigh: generic solving of text-based captchas. In: Proceedings of the 8th USENIX conference on Offensive Technologies, USENIX Association, p 3"},{"issue":"1\u20132","key":"9505_CR31","doi-asserted-by":"crossref","first-page":"57","DOI":"10.1016\/S0004-3702(01)00129-1","volume":"134","author":"M Campbell","year":"2002","unstructured":"Campbell M, Hoane AJ, Hsu F (2002) Deep Blue. Artif Intell 134(1\u20132):57\u201383","journal-title":"Artif Intell"},{"key":"9505_CR32","doi-asserted-by":"crossref","DOI":"10.7551\/mitpress\/9320.001.0001","volume-title":"Developmental robotics: from babies to robots","author":"A Cangelosi","year":"2015","unstructured":"Cangelosi A, Schlesinger M, Smith LB (2015) Developmental robotics: from babies to robots. MIT Press, Cambridge"},{"key":"9505_CR33","doi-asserted-by":"crossref","unstructured":"Caputo B, M\u00fcller H, Martinez-Gomez J, Villegas M, Acar B, Patricia N, Marvasti N, \u00dcsk\u00fcdarl\u0131 S, Paredes R, Cazorla M et al (2014) Imageclef 2014: overview and analysis of the results. In: Information access evaluation. Multilinguality, multimodality, and interaction, Springer, pp 192\u2013211","DOI":"10.1007\/978-3-319-11382-1_18"},{"key":"9505_CR34","doi-asserted-by":"crossref","unstructured":"Carlson A, Betteridge J, Kisiel B, Settles B, Hruschka ER Jr, Mitchell TM (2010) Toward an architecture for never-ending language learning. In: AAAI, vol 5, p 3","DOI":"10.1609\/aaai.v24i1.7519"},{"key":"9505_CR35","doi-asserted-by":"crossref","DOI":"10.1017\/CBO9780511571312","volume-title":"Human cognitive abilities: a survey of factor-analytic studies","author":"JB Carroll","year":"1993","unstructured":"Carroll JB (1993) Human cognitive abilities: a survey of factor-analytic studies. Cambridge University Press, Cambridge"},{"issue":"1","key":"9505_CR36","doi-asserted-by":"crossref","first-page":"41","DOI":"10.1023\/A:1007379606734","volume":"28","author":"R Caruana","year":"1997","unstructured":"Caruana R (1997) Multitask learning. Mach Learn 28(1):41\u201375","journal-title":"Mach Learn"},{"issue":"12","key":"9505_CR37","doi-asserted-by":"crossref","first-page":"941","DOI":"10.1007\/BF02084159","volume":"21","author":"GJ Chaitin","year":"1982","unstructured":"Chaitin GJ (1982) G\u00f6del\u2019s theorem and information. Int J Theor Phys 21(12):941\u2013954","journal-title":"Int J Theor Phys"},{"key":"9505_CR38","doi-asserted-by":"crossref","unstructured":"Chandrasekaran B (1990) What kind of information processing is intelligence? In: The foundation of artificial intelligence\u2014a sourcebook. Cambridge University Press, pp 14\u201346","DOI":"10.1017\/CBO9780511663116.004"},{"issue":"2","key":"9505_CR39","doi-asserted-by":"crossref","first-page":"273","DOI":"10.1080\/713755819","volume":"52","author":"N Chater","year":"1999","unstructured":"Chater N (1999) The search for simplicity: a fundamental cognitive principle? Q J Exp Psychol Sect A 52(2):273\u2013302","journal-title":"Q J Exp Psychol Sect A"},{"issue":"1","key":"9505_CR40","doi-asserted-by":"crossref","first-page":"19","DOI":"10.1016\/S1364-6613(02)00005-0","volume":"7","author":"N Chater","year":"2003","unstructured":"Chater N, Vit\u00e1nyi P (2003) Simplicity: a unifying principle in cognitive science? Trends Cogn Sci 7(1):19\u201322","journal-title":"Trends Cogn Sci"},{"key":"9505_CR41","doi-asserted-by":"crossref","unstructured":"Chu Z, Gianvecchio S, Wang H, Jajodia S (2010) Who is tweeting on twitter: human, bot, or cyborg? In: Proceedings of the 26th annual computer security applications conference, ACM, pp 21\u201330","DOI":"10.1145\/1920261.1920265"},{"key":"9505_CR42","volume-title":"Sampling techniques","author":"WG Cochran","year":"2007","unstructured":"Cochran WG (2007) Sampling techniques. Wiley, New York"},{"issue":"4","key":"9505_CR43","first-page":"35","volume":"9","author":"PR Cohen","year":"1988","unstructured":"Cohen PR, Howe AE (1988) How evaluation guides AI research: the message still counts more than the medium. AI Mag 9(4):35","journal-title":"AI Mag"},{"key":"9505_CR44","unstructured":"Cohen Y (2013) Testing and cognitive enhancement. Technical repor, National Institute for Testing and Evaluation, Jerusalem, Israel"},{"key":"9505_CR45","doi-asserted-by":"crossref","unstructured":"Conrad JG, Zeleznikow J (2013) The significance of evaluation in AI and law: a case study re-examining ICAIL proceedings. In: Proceedings of the 14th international conference on artificial intelligence and law, ACM, pp 186\u2013191","DOI":"10.1145\/2514601.2514624"},{"key":"9505_CR46","doi-asserted-by":"crossref","unstructured":"Conrad JG, Zeleznikow J (2015) The role of evaluation in ai and law. In: Proceedings of the 15th international conference on artificial intelligence and law, pp 181\u2013186","DOI":"10.1145\/2746090.2746116"},{"issue":"5","key":"9505_CR47","doi-asserted-by":"crossref","first-page":"389","DOI":"10.1016\/S0160-2896(01)00062-9","volume":"29","author":"IJ Deary","year":"2001","unstructured":"Deary IJ, Der G, Ford G (2001) Reaction times and intelligence differences: a population-based cohort study. Intelligence 29(5):389\u2013399","journal-title":"Intelligence"},{"key":"9505_CR48","first-page":"487","volume":"2","author":"KS Decker","year":"1989","unstructured":"Decker KS, Durfee EH, Lesser VR (1989) Evaluating research in cooperative distributed problem solving. Distrib Artif Intell 2:487\u2013519","journal-title":"Distrib Artif Intell"},{"key":"9505_CR49","first-page":"1","volume":"7","author":"J Dem\u0161ar","year":"2006","unstructured":"Dem\u0161ar J (2006) Statistical comparisons of classifiers over multiple data sets. J Mach Learn Res 7:1\u201330","journal-title":"J Mach Learn Res"},{"issue":"2\u20133","key":"9505_CR50","doi-asserted-by":"crossref","first-page":"77","DOI":"10.1016\/j.intell.2011.02.006","volume":"39","author":"DK Detterman","year":"2011","unstructured":"Detterman DK (2011) A challenge to Watson. Intelligence 39(2\u20133):77\u201378","journal-title":"Intelligence"},{"key":"9505_CR51","unstructured":"Dimitrakakis C (2016) Personal communication"},{"issue":"3","key":"9505_CR52","doi-asserted-by":"crossref","first-page":"61","DOI":"10.1609\/aimag.v35i3.2548","volume":"35","author":"C Dimitrakakis","year":"2014","unstructured":"Dimitrakakis C, Li G, Tziortziotis N (2014) The reinforcement learning competition 2014. AI Mag 35(3):61\u201365","journal-title":"AI Mag"},{"key":"9505_CR53","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1007\/978-3-642-44958-1","volume-title":"Algorithmic probability and friends. Bayesian prediction and artificial intelligence, lecture notes in computer science","author":"DL Dowe","year":"2013","unstructured":"Dowe DL (2013) Introduction to Ray Solomonoff 85th memorial conference. In: Dowe DL (ed) Algorithmic probability and friends. Bayesian prediction and artificial intelligence, lecture notes in computer science, vol 7070. Springer, Berlin, pp 1\u201336"},{"key":"9505_CR54","unstructured":"Dowe DL, Hajek AR (1997) A computational extension to the Turing Test. In: Proceedings of the 4th conference of the Australasian cognitive science society, University of Newcastle, NSW, Australia"},{"key":"9505_CR55","unstructured":"Dowe DL, Hajek AR (1998) A non-behavioural, computational extension to the Turing test. In: International conference on computational intelligence and multimedia applications (ICCIMA\u201998), Gippsland, Australia, pp 101\u2013106"},{"issue":"2","key":"9505_CR56","doi-asserted-by":"crossref","first-page":"77","DOI":"10.1016\/j.intell.2011.12.001","volume":"40","author":"DL Dowe","year":"2012","unstructured":"Dowe DL, Hern\u00e1ndez-Orallo J (2012) IQ tests are not for machines, yet. Intelligence 40(2):77\u201381","journal-title":"Intelligence"},{"issue":"1","key":"9505_CR57","doi-asserted-by":"crossref","first-page":"51","DOI":"10.1177\/1059712313500502","volume":"22","author":"DL Dowe","year":"2014","unstructured":"Dowe DL, Hern\u00e1ndez-Orallo J (2014) How universal can an intelligence test be? Adapt Behav 22(1):51\u201369","journal-title":"Adapt Behav"},{"key":"9505_CR58","unstructured":"Drummond C (2009) Replicability is not reproducibility: nor is it good science. In: Proceedings of the evaluation methods for machine learning workshop at the 26th ICML, Montreal, Canada"},{"issue":"1","key":"9505_CR59","doi-asserted-by":"crossref","first-page":"67","DOI":"10.1080\/09528130903010295","volume":"22","author":"C Drummond","year":"2010","unstructured":"Drummond C, Japkowicz N (2010) Warning: statistical benchmarking is addictive. Kicking the habit in machine learning. J Exp Theor Artif Intell 22(1):67\u201380","journal-title":"J Exp Theor Artif Intell"},{"key":"9505_CR60","unstructured":"Duan Y, Chen X, Houthooft R, Schulman J, Abbeel P (2016) Benchmarking deep reinforcement learning for continuous control. arXiv preprint arXiv:1604.06778"},{"key":"9505_CR61","volume-title":"Singularity hypotheses: a scientific and philosophical assessment","author":"AH Eden","year":"2013","unstructured":"Eden AH, Moor JH, Soraker JH, Steinhart E (2013) Singularity hypotheses: a scientific and philosophical assessment. Springer, New York"},{"key":"9505_CR62","doi-asserted-by":"crossref","first-page":"37","DOI":"10.1016\/j.actaastro.2011.12.001","volume":"78","author":"W Edmondson","year":"2012","unstructured":"Edmondson W (2012) The intelligence in ETI\u2014what can we know? Acta Astronaut 78:37\u201342","journal-title":"Acta Astronaut"},{"key":"9505_CR63","volume-title":"The rating of chessplayers, past and present","author":"AE Elo","year":"1978","unstructured":"Elo AE (1978) The rating of chessplayers, past and present, vol 3. Batsford, London"},{"key":"9505_CR64","volume-title":"Item response theory for psychologists","author":"SE Embretson","year":"2000","unstructured":"Embretson SE, Reise SP (2000) Item response theory for psychologists. L. Erlbaum, Hillsdale"},{"key":"9505_CR65","unstructured":"Evans JM, Messina ER (2001) Performance metrics for intelligent systems. NIST Special Publication SP, pp 101\u2013104"},{"key":"9505_CR66","doi-asserted-by":"crossref","unstructured":"Everitt T, Lattimore T, Hutter M (2014) Free lunch for optimisation under the universal distribution. In: 2014 IEEE Congress on evolutionary computation (CEC), IEEE, pp 167\u2013174","DOI":"10.1109\/CEC.2014.6900546"},{"issue":"3","key":"9505_CR67","doi-asserted-by":"crossref","first-page":"281","DOI":"10.1023\/A:1009617801681","volume":"4","author":"E Falkenauer","year":"1998","unstructured":"Falkenauer E (1998) On method overfitting. J Heuristics 4(3):281\u2013287","journal-title":"J Heuristics"},{"issue":"1","key":"9505_CR68","first-page":"9","volume":"38","author":"J Feldman","year":"2003","unstructured":"Feldman J (2003) Simplicity and complexity in human concept learning. Gen Psychol 38(1):9\u201315","journal-title":"Gen Psychol"},{"issue":"1","key":"9505_CR69","doi-asserted-by":"crossref","first-page":"9","DOI":"10.1177\/0146621608314608","volume":"33","author":"PJ Ferrando","year":"2009","unstructured":"Ferrando PJ (2009) Difficulty, discrimination, and information indices in the linear factor analysis model for continuous item responses. Appl Psychol Meas 33(1):9\u201324","journal-title":"Appl Psychol Meas"},{"key":"9505_CR70","first-page":"111","volume":"33","author":"PJ Ferrando","year":"2012","unstructured":"Ferrando PJ (2012) Assessing the discriminating power of item and test scores in the linear factor-analysis model. Psicol\u00f3gica 33:111\u2013139","journal-title":"Psicol\u00f3gica"},{"issue":"1","key":"9505_CR71","doi-asserted-by":"crossref","first-page":"27","DOI":"10.1016\/j.patrec.2008.08.010","volume":"30","author":"C Ferri","year":"2009","unstructured":"Ferri C, Hern\u00e1ndez-Orallo J, Modroiu R (2009) An experimental comparison of performance measures for classification. Pattern Recogn Lett 30(1):27\u201338","journal-title":"Pattern Recogn Lett"},{"issue":"3","key":"9505_CR72","doi-asserted-by":"crossref","first-page":"59","DOI":"10.1609\/aimag.v31i3.2303","volume":"31","author":"D Ferrucci","year":"2010","unstructured":"Ferrucci D, Brown E, Chu-Carroll J, Fan J, Gondek D, Kalyanpur AA, Lally A, Murdock J, Nyberg E, Prager J et al (2010) Building Watson: an overview of the DeepQA project. AI Mag 31(3):59\u201379","journal-title":"AI Mag"},{"issue":"2","key":"9505_CR73","doi-asserted-by":"crossref","first-page":"223","DOI":"10.1080\/01969729108902281","volume":"22","author":"DB Fogel","year":"1991","unstructured":"Fogel DB (1991) The evolution of intelligent decision making in gaming. Cybern Syst 22(2):223\u2013236","journal-title":"Cybern Syst"},{"key":"9505_CR74","first-page":"241","volume":"1","author":"J Gaschnig","year":"1983","unstructured":"Gaschnig J, Klahr P, Pople H, Shortliffe E, Terry A (1983) Evaluation of expert systems: issues and case studies. Build Exp Syst 1:241\u2013278","journal-title":"Build Exp Syst"},{"issue":"2","key":"9505_CR75","first-page":"26","volume":"3","author":"JR Geissman","year":"1988","unstructured":"Geissman JR, Schultz RD (1988) Verification & validation. AI Exp 3(2):26\u201333","journal-title":"AI Exp"},{"issue":"2","key":"9505_CR76","first-page":"62","volume":"26","author":"M Genesereth","year":"2005","unstructured":"Genesereth M, Love N, Pell B (2005) General game playing: overview of the AAAI competition. AI Mag 26(2):62","journal-title":"AI Mag"},{"key":"9505_CR77","doi-asserted-by":"crossref","unstructured":"Ger\u00f3nimo D, L\u00f3pez AM (2014) Datasets and benchmarking. In: Vision-based pedestrian protection systems for intelligent vehicles. Springer, pp 87\u201393","DOI":"10.1007\/978-1-4614-7987-1_5"},{"key":"9505_CR78","volume-title":"Artificial general intelligence","year":"2007","unstructured":"Goertzel B, Pennachin C (eds) (2007) Artificial general intelligence. Springer, New York"},{"key":"9505_CR79","unstructured":"Goertzel B, Arel I, Scheutz M (2009) Toward a roadmap for human-level artificial general intelligence: embedding HLAI systems in broad, approachable, physical or virtual contexts. Artif Gen Intell Roadmap Initiat"},{"issue":"4","key":"9505_CR80","doi-asserted-by":"crossref","first-page":"325","DOI":"10.1007\/s00037-007-0232-y","volume":"16","author":"O Goldreich","year":"2007","unstructured":"Goldreich O, Vadhan S (2007) Special issue on worst-case versus average-case complexity editors\u2019 foreword. Comput complex 16(4):325\u2013330","journal-title":"Comput complex"},{"key":"9505_CR81","doi-asserted-by":"crossref","unstructured":"Gordon BB (2007) Report on panel discussion on (re-)establishing or increasing collaborative links between artificial intelligence and intelligent systems. In: Messina ER, Madhavan R (eds) Proceedings of the 2007 workshop on performance metrics for intelligent systems, pp 302\u2013303","DOI":"10.1145\/1660877.1660919"},{"issue":"11","key":"9505_CR82","doi-asserted-by":"crossref","first-page":"90","DOI":"10.1145\/2736282","volume":"58","author":"S Gulwani","year":"2015","unstructured":"Gulwani S, Hern\u00e1ndez-Orallo J, Kitzelmann E, Muggleton SH, Schmid U, Zorn B (2015) Inductive programming meets the real world. Commun ACM 58(11):90\u201399","journal-title":"Commun ACM"},{"key":"9505_CR83","volume-title":"Measurement theory and practice","author":"DJ Hand","year":"2004","unstructured":"Hand DJ (2004) Measurement theory and practice. A Hodder Arnold Publication, London"},{"issue":"4","key":"9505_CR84","doi-asserted-by":"crossref","first-page":"447","DOI":"10.1023\/A:1008367325700","volume":"9","author":"J Hern\u00e1ndez-Orallo","year":"2000","unstructured":"Hern\u00e1ndez-Orallo J (2000a) Beyond the Turing test. J Logic Lang Inf 9(4):447\u2013466","journal-title":"J Logic Lang Inf"},{"key":"9505_CR85","first-page":"1","volume-title":"Performance metrics for intelligent systems workshop","author":"J Hern\u00e1ndez-Orallo","year":"2000","unstructured":"Hern\u00e1ndez-Orallo J (2000b) On the computational measurement of intelligence factors. In: Meystel A (ed) Performance metrics for intelligent systems workshop. National Institute of Standards and Technology, Gaithersburg, pp 1\u20138"},{"issue":"1","key":"9505_CR86","first-page":"49","volume":"13","author":"J Hern\u00e1ndez-Orallo","year":"2000","unstructured":"Hern\u00e1ndez-Orallo J (2000c) Thesis: computational measures of information gain and reinforcement in inference processes. AI Commun 13(1):49\u201350","journal-title":"AI Commun"},{"key":"9505_CR87","unstructured":"Hern\u00e1ndez-Orallo J (2010) A (hopefully) non-biased universal environment class for measuring intelligence of biological and artificial systems. In: Artificial general intelligence, 3rd International Conference. Atlantis Press, Extended report at http:\/\/users.dsic.upv.es\/proy\/anynt\/unbiased.pdf , pp 182\u2013183"},{"key":"9505_CR88","doi-asserted-by":"publisher","unstructured":"Hern\u00e1ndez-Orallo J (2014) On environment difficulty and discriminating power. Auton Agents Multi-Agent Syst. 29(3):402\u2013454. doi: 10.1007\/s10458-014-9257-1","DOI":"10.1007\/s10458-014-9257-1"},{"issue":"18","key":"9505_CR89","doi-asserted-by":"crossref","first-page":"1508","DOI":"10.1016\/j.artint.2010.09.006","volume":"174","author":"J Hern\u00e1ndez-Orallo","year":"2010","unstructured":"Hern\u00e1ndez-Orallo J, Dowe DL (2010) Measuring universal intelligence: towards an anytime intelligence test. Artif Intell 174(18):1508\u20131539","journal-title":"Artif Intell"},{"key":"9505_CR90","doi-asserted-by":"crossref","first-page":"179","DOI":"10.1007\/s11023-012-9299-6","volume":"23","author":"J Hern\u00e1ndez-Orallo","year":"2013","unstructured":"Hern\u00e1ndez-Orallo J, Dowe DL (2013) On potential cognitive abilities in the machine kingdom. Minds Mach 23:179\u2013210","journal-title":"Minds Mach"},{"key":"9505_CR91","unstructured":"Hern\u00e1ndez-Orallo J, Minaya-Collado N (1998) A formal definition of intelligence based on an intensional variant of Kolmogorov complexity. In: Proceedings of international symposium of engineering of intelligent systems (EIS\u201998), ICSC Press, pp 146\u2013163"},{"key":"9505_CR92","doi-asserted-by":"crossref","first-page":"82","DOI":"10.1007\/978-3-642-22887-2_9","volume-title":"Artificial general intelligence, LNAI","author":"J Hern\u00e1ndez-Orallo","year":"2011","unstructured":"Hern\u00e1ndez-Orallo J, Dowe DL, Espa\u00f1a-Cubillo S, Hern\u00e1ndez-Lloreda MV, Insa-Cabrera J (2011) On more realistic environment distributions for defining, evaluating and developing intelligence. In: Schmidhuber J, Th\u00f3risson K, Looks M (eds) Artificial general intelligence, LNAI, vol 6830. Springer, New York, pp 82\u201391"},{"issue":"1","key":"9505_CR93","first-page":"2813","volume":"13","author":"J Hern\u00e1ndez-Orallo","year":"2012","unstructured":"Hern\u00e1ndez-Orallo J, Flach P, Ferri C (2012a) A unified view of performance metrics: translating threshold choice into expected classification loss. J Mach Learn Res 13(1):2813\u20132869","journal-title":"J Mach Learn Res"},{"key":"9505_CR94","doi-asserted-by":"crossref","unstructured":"Hern\u00e1ndez-Orallo J, Insa-Cabrera J, Dowe DL, Hibbard B (2012b) Turing Tests with Turing machines. In: Voronkov A (ed) Turing-100, EPiC Series, vol\u00a010, pp 140\u2013156","DOI":"10.29007\/9n7d"},{"key":"9505_CR95","doi-asserted-by":"crossref","first-page":"50","DOI":"10.1016\/j.cogsys.2013.06.001","volume":"27","author":"J Hern\u00e1ndez-Orallo","year":"2014","unstructured":"Hern\u00e1ndez-Orallo J, Dowe DL, Hern\u00e1ndez-Lloreda MV (2014) Universal psychometrics: measuring cognitive abilities in the machine kingdom. Cogn Syst Res 27:50\u201374","journal-title":"Cogn Syst Res"},{"key":"9505_CR96","doi-asserted-by":"crossref","first-page":"74","DOI":"10.1016\/j.artint.2015.09.011","volume":"230","author":"J Hern\u00e1ndez-Orallo","year":"2016","unstructured":"Hern\u00e1ndez-Orallo J, Mart\u00ednez-Plumed F, Schmid U, Siebers M, Dowe DL (2016) Computer models solving intelligence test problems: progress and implications. Artif Intell 230:74\u2013107","journal-title":"Artif Intell"},{"issue":"5843","key":"9505_CR97","doi-asserted-by":"crossref","first-page":"1360","DOI":"10.1126\/science.1146282","volume":"317","author":"E Herrmann","year":"2007","unstructured":"Herrmann E, Call J, Hern\u00e1ndez-Lloreda MV, Hare B, Tomasello M (2007) Humans have evolved specialized skills of social cognition: the cultural intelligence hypothesis. Science 317(5843):1360\u20131366","journal-title":"Science"},{"issue":"1","key":"9505_CR98","doi-asserted-by":"crossref","first-page":"54","DOI":"10.2478\/v10229-011-0004-6","volume":"1","author":"B Hibbard","year":"2009","unstructured":"Hibbard B (2009) Bias and no free lunch in formal measures of intelligence. J Artif Gen Intell 1(1):54\u201361","journal-title":"J Artif Gen Intell"},{"key":"9505_CR99","doi-asserted-by":"crossref","unstructured":"Hingston P (2010) A new design for a Turing Test for bots. In: 2010 IEEE symposium on computational intelligence and games (CIG), IEEE, pp 345\u2013350","DOI":"10.1109\/ITW.2010.5593336"},{"key":"9505_CR100","doi-asserted-by":"crossref","DOI":"10.1007\/978-3-642-32323-2","volume-title":"Believable bots: can computers play like people?","author":"P Hingston","year":"2012","unstructured":"Hingston P (2012) Believable bots: can computers play like people?. Springer, New York"},{"issue":"3","key":"9505_CR101","doi-asserted-by":"crossref","first-page":"289","DOI":"10.1109\/34.990132","volume":"24","author":"TK Ho","year":"2002","unstructured":"Ho TK, Basu M (2002) Complexity measures of supervised classification problems. IEEE Trans Pattern Anal Mach Intell 24(3):289\u2013300","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"9505_CR102","doi-asserted-by":"crossref","first-page":"227","DOI":"10.1007\/978-3-540-68677-4_8","volume-title":"Artificial general intelligence, cognitive technologies","author":"M Hutter","year":"2007","unstructured":"Hutter M (2007) Universal algorithmic intelligence: a mathematical top $$\\rightarrow $$ \u2192 down approach. In: Goertzel B, Pennachin C (eds) Artificial general intelligence, cognitive technologies. Springer, Berlin, pp 227\u2013290"},{"issue":"4","key":"9505_CR103","doi-asserted-by":"crossref","first-page":"313","DOI":"10.1007\/s10852-005-2586-y","volume":"3","author":"C Igel","year":"2005","unstructured":"Igel C, Toussaint M (2005) A no-free-lunch theorem for non-uniform distributions of target functions. J Math Model Algorithms 3(4):313\u2013322","journal-title":"J Math Model Algorithms"},{"key":"9505_CR104","unstructured":"Insa-Cabrera J (2016) Towards a universal test of social intelligence. Ph.D. thesis, Departament de Sistemes Inform\u00e1tics i Computaci\u00f3, UPV"},{"key":"9505_CR105","doi-asserted-by":"crossref","first-page":"122","DOI":"10.1007\/978-3-642-22887-2_13","volume-title":"Artificial general intelligence, LNAI","author":"J Insa-Cabrera","year":"2011","unstructured":"Insa-Cabrera J, Dowe DL, Espa\u00f1a-Cubillo S, Hern\u00e1ndez-Lloreda MV, Hern\u00e1ndez-Orallo J (2011a) Comparing humans and ai agents. In: Schmidhuber J, Th\u00f3risson K, Looks M (eds) Artificial general intelligence, LNAI, vol 6830. Springer, New York, pp 122\u2013132"},{"key":"9505_CR106","volume-title":"Current topics in artificial intelligence. CAEPIA 2011, LNAI series 7023","author":"J Insa-Cabrera","year":"2011","unstructured":"Insa-Cabrera J, Dowe DL, Hern\u00e1ndez-Orallo J (2011) Evaluating a reinforcement learning algorithm with a general intelligence test. In: Lozano JA, Gamez JM (eds) Current topics in artificial intelligence. CAEPIA 2011, LNAI series 7023. Springer, New York"},{"key":"9505_CR107","first-page":"126","volume-title":"AGI, lecture notes in computer science","author":"J Insa-Cabrera","year":"2012","unstructured":"Insa-Cabrera J, Benacloch-Ayuso JL, Hern\u00e1ndez-Orallo J (2012) On measuring social intelligence: experiments on competition and cooperation. In: Bach J, Goertzel B, Ikl\u00e9 M (eds) AGI, lecture notes in computer science, vol 7716. Springer, New York, pp 126\u2013135"},{"key":"9505_CR108","doi-asserted-by":"crossref","unstructured":"Jacoff A, Messina E, Weiss BA, Tadokoro S, Nakagawa Y (2003) Test arenas and performance metrics for urban search and rescue robots. In: Proceedings of 2003 IEEE\/RSJ international conference on intelligent robots and systems, 2003 (IROS 2003), IEEE, vol\u00a04, pp 3396\u20133403","DOI":"10.1109\/IROS.2003.1249681"},{"key":"9505_CR109","doi-asserted-by":"crossref","DOI":"10.1017\/CBO9780511921803","volume-title":"Evaluating learning algorithms","author":"N Japkowicz","year":"2011","unstructured":"Japkowicz N, Shah M (2011) Evaluating learning algorithms. Cambridge University Press, Cambridge"},{"key":"9505_CR110","doi-asserted-by":"crossref","unstructured":"Jiang J (2008) A literature survey on domain adaptation of statistical classifiers. http:\/\/sifaka.cs.uiuc.edu\/jiang4\/domain_adaptation\/survey","DOI":"10.1145\/1321440.1321498"},{"key":"9505_CR111","unstructured":"Johnson M, Hofmann K, Hutton T, Bignell D (2016) The Malmo platform for artificial intelligence experimentation. In: International joint conference on artificial intelligence (IJCAI)"},{"issue":"7","key":"9505_CR112","doi-asserted-by":"crossref","first-page":"635","DOI":"10.1002\/pits.20498","volume":"47","author":"TZ Keith","year":"2010","unstructured":"Keith TZ, Reynolds MR (2010) Cattell\u2013Horn\u2013Carroll abilities and cognitive tests: what we\u2019ve learned from 20 years of research. Psychol Schools 47(7):635\u2013650","journal-title":"Psychol Schools"},{"issue":"2","key":"9505_CR113","doi-asserted-by":"crossref","first-page":"103","DOI":"10.1609\/aimag.v33i2.2396","volume":"33","author":"W Ketter","year":"2012","unstructured":"Ketter W, Symeonidis A (2012) Competitive benchmarking: lessons learned from the trading agent competition. AI Mag 33(2):103","journal-title":"AI Mag"},{"key":"9505_CR114","doi-asserted-by":"crossref","first-page":"105","DOI":"10.1016\/j.ins.2012.02.017","volume":"197","author":"W Khreich","year":"2012","unstructured":"Khreich W, Granger E, Miri A, Sabourin R (2012) A survey of techniques for incremental learning of HMM parameters. Inf Sci 197:105\u2013130","journal-title":"Inf Sci"},{"key":"9505_CR115","doi-asserted-by":"crossref","DOI":"10.1007\/b95999","volume-title":"Soccer robotics","author":"JH Kim","year":"2004","unstructured":"Kim JH (2004) Soccer robotics, vol 11. Springer, New York"},{"key":"9505_CR116","doi-asserted-by":"crossref","unstructured":"Kitano H, Asada M, Kuniyoshi Y, Noda I, Osawa E (1997) Robocup: the robot world cup initiative. In: Proceedings of the first international conference on autonomous agents, ACM, pp 340\u2013347","DOI":"10.1145\/267658.267738"},{"issue":"8723, 5 March 2","key":"9505_CR117","first-page":"82","volume":"398","author":"K Kleiner","year":"2011","unstructured":"Kleiner K (2011) Who are you calling bird-brained? An attempt is being made to devise a universal intelligence test. Economist 398(8723, 5 March 2011):82","journal-title":"Economist"},{"key":"9505_CR118","volume-title":"Sorting and searching, volume 3 of the art of computer programming","author":"DE Knuth","year":"1973","unstructured":"Knuth DE (1973) Sorting and searching, volume 3 of the art of computer programming. Addison-Wesley, Reading"},{"issue":"3\u20134","key":"9505_CR119","doi-asserted-by":"crossref","first-page":"251","DOI":"10.1007\/s10710-010-9112-3","volume":"11","author":"JR Koza","year":"2010","unstructured":"Koza JR (2010) Human-competitive results produced by genetic programming. Genet Program Evolvable Mach 11(3\u20134):251\u2013284","journal-title":"Genet Program Evolvable Mach"},{"key":"9505_CR120","first-page":"187","volume-title":"The nature of thought: essays in honor of D. O. Hebb","author":"J Krueger","year":"1980","unstructured":"Krueger J, Osherson D (1980) On the psychology of structural simplicity. In: Jusczyk PW, Klein RM (eds) The nature of thought: essays in honor of D. O. Hebb. Psychology Press, London, pp 187\u2013205"},{"key":"9505_CR121","unstructured":"Langford J (2005) Clever methods of overfitting. Machine Learning (Theory). http:\/\/hunch.net"},{"issue":"3","key":"9505_CR122","first-page":"195","volume":"2","author":"P Langley","year":"1987","unstructured":"Langley P (1987) Research papers in machine learning. Mach Learn 2(3):195\u2013198","journal-title":"Mach Learn"},{"issue":"3","key":"9505_CR123","doi-asserted-by":"crossref","first-page":"275","DOI":"10.1007\/s10994-011-5242-y","volume":"82","author":"P Langley","year":"2011","unstructured":"Langley P (2011) The changing science of machine learning. Mach Learn 82(3):275\u2013279","journal-title":"Mach Learn"},{"key":"9505_CR124","first-page":"3","volume":"1","author":"P Langley","year":"2012","unstructured":"Langley P (2012) The cognitive systems paradigm. Adv Cogn Syst 1:3\u201313","journal-title":"Adv Cogn Syst"},{"key":"9505_CR125","doi-asserted-by":"crossref","unstructured":"Lattimore T, Hutter M (2013) No free lunch versus Occam\u2019s razor in supervised learning. Algorithmic Probability and Friends. Springer, Bayesian Prediction and Artificial Intelligence, pp 223\u2013235","DOI":"10.1007\/978-3-642-44958-1_17"},{"key":"9505_CR126","doi-asserted-by":"crossref","DOI":"10.1017\/CBO9781139342223","volume-title":"Structural information theory: the simplicity of visual form","author":"ELJ Leeuwenberg","year":"2012","unstructured":"Leeuwenberg ELJ, Van Der Helm PA (2012) Structural information theory: the simplicity of visual form. Cambridge University Press, Cambridge"},{"key":"9505_CR127","doi-asserted-by":"publisher","unstructured":"Legg S, Hutter M (2007a) Tests of machine intelligence. In: Lungarella M, Iida F, Bongard J, Pfeifer R (eds) 50 Years of Artificial Intelligence, Lecture Notes in Computer Science, vol 4850, Springer Berlin Heidelberg, pp 232\u2013242. doi: 10.1007\/978-3-540-77296-5_22","DOI":"10.1007\/978-3-540-77296-5_22"},{"issue":"4","key":"9505_CR128","doi-asserted-by":"crossref","first-page":"391","DOI":"10.1007\/s11023-007-9079-x","volume":"17","author":"S Legg","year":"2007","unstructured":"Legg S, Hutter M (2007b) Universal intelligence: a definition of machine intelligence. Minds Mach 17(4):391\u2013444","journal-title":"Minds Mach"},{"key":"9505_CR129","doi-asserted-by":"crossref","unstructured":"Legg S, Veness J (2013) An approximation of the universal intelligence measure. Algorithmic Probability and Friends. Springer, Bayesian Prediction and Artificial Intelligence, pp 236\u2013249","DOI":"10.1007\/978-3-642-44958-1_18"},{"key":"9505_CR130","doi-asserted-by":"crossref","first-page":"27","DOI":"10.1016\/j.artint.2014.03.007","volume":"212","author":"HJ Levesque","year":"2014","unstructured":"Levesque HJ (2014) On our best behaviour. Artif Intell 212:27\u201335","journal-title":"Artif Intell"},{"key":"9505_CR131","unstructured":"Levesque HJ, Davis E, Morgenstern L (2012) The winograd schema challenge. In: Proceedings of the thirteenth international conference on the principles of knowledge representation and reasoning, pp 552\u2013561"},{"issue":"3","key":"9505_CR132","first-page":"265","volume":"9","author":"LA Levin","year":"1973","unstructured":"Levin LA (1973) Universal sequential search problems. Prob Inf Transm 9(3):265\u2013266","journal-title":"Prob Inf Transm"},{"key":"9505_CR133","doi-asserted-by":"crossref","first-page":"285","DOI":"10.1137\/0215020","volume":"15","author":"LA Levin","year":"1986","unstructured":"Levin LA (1986) Average case complete problems. SIAM J Comput 15:285\u2013286","journal-title":"SIAM J Comput"},{"key":"9505_CR134","doi-asserted-by":"crossref","first-page":"53","DOI":"10.1007\/978-3-642-44958-1_3","volume-title":"Algorithmic probability and friends. Bayesian prediction and artificial intelligence, lecture notes in computer science","author":"LA Levin","year":"2013","unstructured":"Levin LA (2013) Universal heuristics: how do humans solve unsolvable problems? In: Dowe DL (ed) Algorithmic probability and friends. Bayesian prediction and artificial intelligence, lecture notes in computer science, vol 7070. Springer, New York, pp 53\u201354"},{"key":"9505_CR135","doi-asserted-by":"crossref","DOI":"10.1007\/978-0-387-49820-1","volume-title":"An introduction to Kolmogorov complexity and its applications","author":"M Li","year":"2008","unstructured":"Li M, Vit\u00e1nyi P (2008) An introduction to Kolmogorov complexity and its applications, 3rd edn. Springer, New York","edition":"3"},{"issue":"1","key":"9505_CR136","doi-asserted-by":"crossref","first-page":"6","DOI":"10.1145\/1111293.1111303","volume":"4","author":"D Livingstone","year":"2006","unstructured":"Livingstone D (2006) Turing\u2019s test and believable AI in games. Comput Entertain CIE 4(1):6","journal-title":"Comput Entertain CIE"},{"key":"9505_CR137","doi-asserted-by":"crossref","unstructured":"Llargues-Asensio JM, Peralta J, Arrabales R, Gonz\u00e1lez-Bed\u00eda M, Cortez P, L\u00f3pez-Pe\u00f1a AL (2014) Artificial intelligence approaches for the generation and assessment of believable human-like behaviour in virtual characters. Expert Systems with Applications","DOI":"10.1016\/j.eswa.2014.05.004"},{"key":"9505_CR138","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1023\/A:1026044009832","volume":"20","author":"D Long","year":"2003","unstructured":"Long D, Fox M (2003) The 3rd international planning competition: results and analysis. J Artif Intell Res JAIR 20:1\u201359","journal-title":"J Artif Intell Res JAIR"},{"key":"9505_CR139","volume-title":"Applications of item response theory to practical testing problems","author":"FM Lord","year":"1980","unstructured":"Lord FM (1980) Applications of item response theory to practical testing problems. Erlbaum, Mahwah"},{"key":"9505_CR140","doi-asserted-by":"crossref","first-page":"237","DOI":"10.1016\/j.ins.2013.08.059","volume":"261","author":"N Maci\u00e0","year":"2014","unstructured":"Maci\u00e0 N, Bernad\u00f3-Mansilla E (2014) Towards UCI+: a mindful repository design. Inf Sci 261:237\u2013262","journal-title":"Inf Sci"},{"key":"9505_CR141","doi-asserted-by":"crossref","DOI":"10.1007\/978-1-4419-0492-8","volume-title":"Performance evaluation and benchmarking of intelligent systems","author":"R Madhavan","year":"2009","unstructured":"Madhavan R, Tunstel E, Messina E (2009) Performance evaluation and benchmarking of intelligent systems. Springer, New York"},{"key":"9505_CR142","unstructured":"Mahoney MV (1999) Text compression as a test for artificial intelligence. In: Proceedings of the national conference on artificial intelligence, AAAI, p 970"},{"key":"9505_CR143","doi-asserted-by":"crossref","unstructured":"March\u00e9 C, Zantema H (2007) The termination competition. In: Term rewriting and applications, Springer, pp 303\u2013313","DOI":"10.1007\/978-3-540-73449-9_23"},{"issue":"1","key":"9505_CR144","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1609\/aimag.v37i1.2650","volume":"37","author":"G Marcus","year":"2016","unstructured":"Marcus G, Rossi F, Veloso M (2016) Beyond the Turing test (special issue). AI Mag 37(1):3\u2013101","journal-title":"AI Mag"},{"key":"9505_CR145","unstructured":"Masum H, Christensen S (2003) The turing ratio: a framework for open-ended task metrics. J Evol Technol"},{"key":"9505_CR146","unstructured":"Masum H, Christensen S, Oppacher F (2002) The turing ratio: metrics for open-ended tasks. In: GECCO, Citeseer, pp 973\u2013980"},{"key":"9505_CR147","unstructured":"McCarthy J (2007) What is artificial intelligence. Technical report, Stanford University. http:\/\/www-formal.stanford.edu\/jmc\/whatisai.html"},{"key":"9505_CR148","doi-asserted-by":"crossref","DOI":"10.1201\/9780429258985","volume-title":"Machines who think","author":"P McCorduck","year":"2004","unstructured":"McCorduck P (2004) Machines who think. A K Peters\/CRC Press, Boca Raton"},{"key":"9505_CR149","doi-asserted-by":"crossref","unstructured":"McDermott J, White DR, Luke S, Manzoni L, Castelli M, Vanneschi L, Ja\u015bkowski W, Krawiec K, Harper R, Jong KD, O\u2019Reilly UM (2012) Genetic programming needs better benchmarks. In: Proceedings of the 14th international conference on Genetic and evolutionary computation conference. ACM, Philadelphia, pp 791\u2013798","DOI":"10.1145\/2330163.2330273"},{"key":"9505_CR150","unstructured":"McGuigan M (2006) Graphics Turing Test. arXiv preprint arXiv:cs\/0603132"},{"issue":"4","key":"9505_CR151","doi-asserted-by":"crossref","first-page":"p43","DOI":"10.5430\/air.v3n4p43","volume":"3","author":"AV Melkikh","year":"2014","unstructured":"Melkikh AV (2014) The no free lunch theorem and hypothesis of instinctive animal behavior. Artif Intell Res 3(4):p43","journal-title":"Artif Intell Res"},{"issue":"2","key":"9505_CR152","doi-asserted-by":"crossref","first-page":"300","DOI":"10.1037\/0033-2909.115.2.300","volume":"115","author":"GJ Mellenbergh","year":"1994","unstructured":"Mellenbergh GJ (1994) Generalized linear item response theory. Psychol Bull 115(2):300","journal-title":"Psychol Bull"},{"key":"9505_CR153","unstructured":"Mesnil G, Dauphin Y, Glorot X, Rifai S, Bengio Y, Goodfellow IJ, Lavoie E, Muller X, Desjardins G, Warde-Farley D, et\u00a0al (2012) Unsupervised and transfer learning challenge: a deep learning approach. JMLR: Workshop and Conference Proceedings, 2012 ICML Workshop on Unsupervised and Transfer Learning vol 27, pp 97\u2013110"},{"key":"9505_CR154","unstructured":"Messina E, Meystel A, Reeker L (2001) PerMIS 2001, white paper. In: Meystel AM, Messina ER (eds) Measuring the performance and intelligence of systems: proceedings of the 2001 PerMIS Workshop, September 4, 2001, National Institute of Standards and Technology (NIST) Special Publication 982. Gaithersburg, pp 3\u201315"},{"key":"9505_CR155","unstructured":"Meystel A (2000) Permis 2000 white paper: measuring performance and intelligence of systems with autonomy. In: Meystel AM, Messina ER (eds) Measuring the performance and intelligence of systems: proceedings of the 2000 PerMIS Workshop, August 14\u201316, 2000, National Institute of Standards and Technology (NIST) Special Publication 970. Gaithersburg, pp 1\u201334"},{"key":"9505_CR156","doi-asserted-by":"crossref","unstructured":"Meystel A, Albus J, Messina E, Leedom D (2003a) Performance measures for intelligent systems: measures of technology readiness. Technical report, DTIC Document","DOI":"10.6028\/NIST.SP.1014"},{"key":"9505_CR157","doi-asserted-by":"crossref","unstructured":"Meystel A, Albus J, Messina E, Leedom D (2003) Permis 2003 white paper: performance measures for intelligent systems\u2014measures of technology readiness. In: Meystel AM, Messina ER (eds) Measuring the performance and intelligence of systems: proceedings of the 2003 PerMIS Workshop, National Institute of Standards and Technology (NIST) Special Publication 1014. Gaithersburg","DOI":"10.6028\/NIST.SP.1014"},{"key":"9505_CR158","volume-title":"Semantic information processing","year":"1968","unstructured":"Minsky ML (ed) (1968) Semantic information processing. MIT Press, Cambridge"},{"issue":"7540","key":"9505_CR159","doi-asserted-by":"crossref","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih V, Kavukcuoglu K, Silver D, Rusu AA, Veness J, Bellemare MG, Graves A, Riedmiller M, Fidjeland AK, Ostrovski G et al (2015) Human-level control through deep reinforcement learning. Nature 518(7540):529\u2013533","journal-title":"Nature"},{"issue":"1","key":"9505_CR160","doi-asserted-by":"crossref","first-page":"50","DOI":"10.1609\/aimag.v37i1.2639","volume":"37","author":"L Morgenstern","year":"2016","unstructured":"Morgenstern L, Davis E, Ortiz-Jr CL (2016) Planning, executing, and evaluating the Winograd schema challenge. AI Mag 37(1):50\u201354","journal-title":"AI Mag"},{"key":"9505_CR161","unstructured":"Mueller S, Jones M, Minnery B, Hiland JM (2007) The bica cognitive decathlon: a test suite for biologically-inspired cognitive agents. In: Proceedings of behavior representation in modeling and simulation conference, Norfolk"},{"issue":"02","key":"9505_CR162","doi-asserted-by":"crossref","first-page":"273","DOI":"10.1142\/S1793843010000497","volume":"2","author":"ST Mueller","year":"2010","unstructured":"Mueller ST (2010) A partial implementation of the BICA cognitive decathlon using the psychology experiment building language (PEBL). Int J Mach Conscious 2(02):273\u2013288","journal-title":"Int J Mach Conscious"},{"key":"9505_CR163","unstructured":"Mueller ST, Minnery BS (2008) Adapting the Turing Test for embodied neurocognitive evaluation of biologically-inspired cognitive agents. In: Proceedings of 2008 AAAI fall symposium on biologically inspired cognitive architectures"},{"key":"9505_CR164","first-page":"283","volume-title":"Vis Inf Process","author":"A Newell","year":"1973","unstructured":"Newell A (1973) You can\u2019t play 20 questions with nature and win: projective comments on the papers of this symposium. In: Chase W (ed) Vis Inf Process. Academic Press, New York, pp 283\u2013308"},{"issue":"2","key":"9505_CR165","doi-asserted-by":"crossref","first-page":"135","DOI":"10.1207\/s15516709cog0402_2","volume":"4","author":"A Newell","year":"1980","unstructured":"Newell A (1980) Physical symbol systems. Cogn Sci 4(2):135\u2013183","journal-title":"Cogn Sci"},{"key":"9505_CR166","volume-title":"Unified theories of cognition","author":"A Newell","year":"1990","unstructured":"Newell A (1990) Unified theories of cognition. Harvard University, Cambridge"},{"issue":"3","key":"9505_CR167","doi-asserted-by":"crossref","first-page":"113","DOI":"10.1145\/360018.360022","volume":"19","author":"A Newell","year":"1976","unstructured":"Newell A, Simon HA (1976) Computer science as empirical inquiry: symbols and search. Commun ACM 19(3):113\u2013126","journal-title":"Commun ACM"},{"key":"9505_CR168","unstructured":"Nizamani AR (2015) Reasoning with bounded cognitive resources. Ph.D. thesis, Department of Applied Information Technology, Chalmers University of Technology & University of Gothenburg, Sweden"},{"key":"9505_CR169","unstructured":"Oppy G, Dowe DL (2011) The Turing Test. In: Zalta EN (ed) Stanford Encyclopedia of Philosophy, Stanford University. http:\/\/plato.stanford.edu\/entries\/turing-test\/"},{"issue":"10","key":"9505_CR170","doi-asserted-by":"crossref","first-page":"1345","DOI":"10.1109\/TKDE.2009.191","volume":"22","author":"SJ Pan","year":"2010","unstructured":"Pan SJ, Yang Q (2010) A survey on transfer learning. IEEE Trans Knowl Data Eng 22(10):1345\u20131359","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"9505_CR171","unstructured":"Perez D, Samothrakis S, Togelius J, Schaul T, Lucas S, Cou\u00ebtoux A, Lee J, Lim CU, Thompson T (2015) The 2014 general video game playing competition. IEEE Transactions on Computational Intelligence and AI in Games"},{"key":"9505_CR172","unstructured":"Potthast M, Hagen M, Gollub T, Tippmann M, Kiesel J, Rosso P, Stamatatos E, Stein B (2013) Overview of the 5th international competition on plagiarism detection. CLEF (2013) Evaluation labs and workshop working notes papers, pp 23\u201326 September. Valencia, Spain"},{"issue":"5","key":"9505_CR173","doi-asserted-by":"crossref","first-page":"950","DOI":"10.1016\/j.artint.2011.01.006","volume":"175","author":"D Proudfoot","year":"2011","unstructured":"Proudfoot D (2011) Anthropomorphism and AI: Turing\u2019s much misunderstood imitation game. Artif Intell 175(5):950\u2013957","journal-title":"Artif Intell"},{"key":"9505_CR174","doi-asserted-by":"crossref","unstructured":"Quinn AJ, Bederson BB (2011) Human computation: a survey and taxonomy of a growing field. In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems, ACM, pp 1403\u20131412","DOI":"10.1145\/1978942.1979148"},{"issue":"1","key":"9505_CR175","first-page":"173","volume":"4","author":"S Rajani","year":"2011","unstructured":"Rajani S (2011) Artificial intelligence\u2014man or machine. Int J Inf Technol 4(1):173\u2013176","journal-title":"Int J Inf Technol"},{"key":"9505_CR176","doi-asserted-by":"crossref","unstructured":"Rao RB, Fung G, Rosales R (2008) On the dangers of cross-validation. an experimental evaluation. In: SDM, SIAM, pp 588\u2013596","DOI":"10.1137\/1.9781611972788.54"},{"issue":"1","key":"9505_CR177","doi-asserted-by":"crossref","first-page":"1","DOI":"10.2478\/v10229-011-0005-5","volume":"2","author":"B Rohrer","year":"2010","unstructured":"Rohrer B (2010) Accelerating progress in artificial general intelligence: choosing a benchmark for natural world interaction. J Artif Gen Intell 2(1):1\u201328","journal-title":"J Artif Gen Intell"},{"key":"9505_CR178","unstructured":"Rothenberg J, Paul J, Kameny I, Kipps JR, Swenson M (1987) Evaluating expert system tools: a framework and methodology-workshops. Technical report, DTIC Document"},{"key":"9505_CR179","volume-title":"Artificial intelligence: a modern approach","author":"S Russell","year":"2009","unstructured":"Russell S, Norvig P (2009) Artificial intelligence: a modern approach. Prentice Hall, Upper Saddle River"},{"key":"9505_CR180","unstructured":"Sanghi P, Dowe DL (2003) A computer program capable of passing IQ tests. In: 4th international conference on cognitive science (ICCS\u201903), Sydney, pp 570\u2013575"},{"issue":"5844","key":"9505_CR181","doi-asserted-by":"crossref","first-page":"1518","DOI":"10.1126\/science.1144079","volume":"317","author":"J Schaeffer","year":"2007","unstructured":"Schaeffer J, Burch N, Bjornsson Y, Kishimoto A, Muller M, Lake R, Lu P, Sutphen S (2007) Checkers is solved. Science 317(5844):1518","journal-title":"Science"},{"key":"9505_CR182","doi-asserted-by":"crossref","unstructured":"Schaie KW (2010) Primary mental abilities. Corsini Encyclopedia of Psychology","DOI":"10.1002\/9780470479216.corpsy0706"},{"key":"9505_CR183","doi-asserted-by":"publisher","unstructured":"Schaul T (2014) An extensible description language for video games. IEEE Trans Comput Intell AI Games PP(99):1\u20131. doi: 10.1109\/TCIAIG.2014.2352795","DOI":"10.1109\/TCIAIG.2014.2352795"},{"key":"9505_CR184","unstructured":"Schenck C (2013) Intelligence tests for robots: Solving perceptual reasoning tasks with a humanoid robot. Master\u2019s thesis, Iowa State University"},{"key":"9505_CR185","unstructured":"Schlenoff C, Scott H, Balakirsky S (2011) Performance evaluation of intelligent systems at the National Institute of Standards and Technology (NIST). Technical report, DTIC Document"},{"key":"9505_CR186","doi-asserted-by":"crossref","unstructured":"Schmid U, Ragni M (2015) Comparing computer models solving number series problems. In: Artificial general intelligence. Springer, pp 352\u2013361","DOI":"10.1007\/978-3-319-21365-1_36"},{"issue":"2","key":"9505_CR187","doi-asserted-by":"crossref","first-page":"263","DOI":"10.1023\/A:1008229619541","volume":"8","author":"P Schweizer","year":"1998","unstructured":"Schweizer P (1998) The truly total Turing test. Minds Mach 8(2):263\u2013272","journal-title":"Minds Mach"},{"key":"9505_CR188","doi-asserted-by":"crossref","first-page":"417","DOI":"10.1017\/S0140525X00005756","volume":"3","author":"JR Searle","year":"1980","unstructured":"Searle JR (1980) Minds, brains, and programs. Behav Brain Sci 3:417\u2013457","journal-title":"Behav Brain Sci"},{"key":"9505_CR189","doi-asserted-by":"crossref","unstructured":"Seber GAF, Salehi MM (2013) Adaptive cluster sampling. In: Adaptive sampling designs. Springer, pp 11\u201326","DOI":"10.1007\/978-3-642-33657-7"},{"issue":"1","key":"9505_CR190","doi-asserted-by":"crossref","first-page":"1","DOI":"10.2200\/S00429ED1V01Y201207AIM018","volume":"6","author":"B Settles","year":"2012","unstructured":"Settles B (2012) Active learning. Synth Lect Artif Intell Mach Learn 6(1):1\u2013114","journal-title":"Synth Lect Artif Intell Mach Learn"},{"key":"9505_CR191","volume-title":"Cognition, evolution, and behavior","author":"SJ Shettleworth","year":"2010","unstructured":"Shettleworth SJ (2010) Cognition, evolution, and behavior. Oxford University Press, Oxford"},{"key":"9505_CR192","volume-title":"Fundamentals of comparative cognition","author":"SJ Shettleworth","year":"2013","unstructured":"Shettleworth SJ, Bloom P, Nadel L (2013) Fundamentals of comparative cognition. Oxford University Press, Oxford"},{"issue":"1","key":"9505_CR193","doi-asserted-by":"crossref","first-page":"91","DOI":"10.1609\/aimag.v37i1.2646","volume":"37","author":"SM Shieber","year":"2016","unstructured":"Shieber SM (2016) Principles for designing an AI competition, or why the Turing test fails as an inducement prize. AI Mag 37(1):91\u201396","journal-title":"AI Mag"},{"issue":"7587","key":"9505_CR194","doi-asserted-by":"crossref","first-page":"484","DOI":"10.1038\/nature16961","volume":"529","author":"D Silver","year":"2016","unstructured":"Silver D, Huang A, Maddison CJ, Guez A, Sifre L, Van Den Driessche G, Schrittwieser J, Antonoglou I, Panneershelvam V, Lanctot M et al (2016) Mastering the game of go with deep neural networks and tree search. Nature 529(7587):484\u2013489","journal-title":"Nature"},{"key":"9505_CR195","unstructured":"Simmons R (2000) Survivability and competence as measures of intelligent systems. In: Meystel AM, Messina ER (eds) Measuring the performance and intelligence of systems: proceedings of the 2000 PerMIS Workshop, August 14\u201316, 2000, National Institute of Standards and Technology (NIST) Special Publication 970. Gaithersburg, pp 162\u2013163"},{"issue":"1","key":"9505_CR196","doi-asserted-by":"crossref","first-page":"95","DOI":"10.1016\/0004-3702(95)00039-H","volume":"77","author":"HA Simon","year":"1995","unstructured":"Simon HA (1995) Artificial intelligence: an empirical science. Artif Intell 77(1):95\u2013127","journal-title":"Artif Intell"},{"key":"9505_CR197","unstructured":"Sloman A, Scheutz M (2002) A framework for comparing agent architectures. Proceedings of UKCI 2"},{"key":"9505_CR198","unstructured":"Smith WD (2002) Rating systems for gameplayers, and learning. NEC, Princeton, NJ, Technical report, pp 93\u2013104"},{"key":"9505_CR199","unstructured":"Smith WD (2006) Mathematical definition of \u201cintelligence\u201d (and consequences). Unpublished report"},{"key":"9505_CR200","doi-asserted-by":"crossref","unstructured":"Soares C (2009) UCI++: improved support for algorithm selection using datasetoids. In: Advances in knowledge discovery and data mining. Springer, pp 499\u2013506","DOI":"10.1007\/978-3-642-01307-2_46"},{"key":"9505_CR201","unstructured":"Solomonoff R (1996) Does algorithmic probability solve the problem of induction. Inf Stat Induction Sci 7\u20138"},{"issue":"1","key":"9505_CR202","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/S0019-9958(64)90223-2","volume":"7","author":"RJ Solomonoff","year":"1964","unstructured":"Solomonoff RJ (1964) A formal theory of inductive inference. Part I. Inf Control 7(1):1\u201322","journal-title":"Inf Control"},{"key":"9505_CR203","unstructured":"Solomonoff RJ (1984) Optimum sequential search. Oxbridge Research, Cambridge. http:\/\/world.std.com\/~rjs\/optseq.pdf"},{"key":"9505_CR204","doi-asserted-by":"crossref","DOI":"10.1007\/978-3-662-05052-1","volume-title":"Importance sampling: applications in communications and detection","author":"R Srinivasan","year":"2002","unstructured":"Srinivasan R (2002) Importance sampling: applications in communications and detection. Springer, New York"},{"key":"9505_CR205","doi-asserted-by":"crossref","unstructured":"Starkie B, van Zaanen M, Estival D (2006) The Tenjinno machine translation competition. In: Grammatical inference: algorithms and applications. Springer, pp 214\u2013226","DOI":"10.1007\/11872436_18"},{"key":"9505_CR206","volume-title":"Handbook of intelligence","year":"2000","unstructured":"Sternberg RJ (ed) (2000) Handbook of intelligence. Cambridge University Press, Cambridge"},{"key":"9505_CR207","doi-asserted-by":"crossref","first-page":"27","DOI":"10.1016\/j.cogsys.2012.05.003","volume":"22\u201323","author":"C Stranneg\u00e5rd","year":"2013","unstructured":"Stranneg\u00e5rd C, Amirghasemi M, Ulfsb\u00fccker S (2013a) An anthropomorphic method for number sequence problems. Cogn Syst Res 22\u201323:27\u201334","journal-title":"Cogn Syst Res"},{"key":"9505_CR208","doi-asserted-by":"crossref","first-page":"130","DOI":"10.1007\/978-3-642-39521-5_14","volume-title":"Artificial general intelligence. Lecture notes in computer science","author":"C Stranneg\u00e5rd","year":"2013","unstructured":"Stranneg\u00e5rd C, Nizamani A, Sj\u00f6berg A, Engstr\u00f6m F (2013b) Bounded Kolmogorov complexity based on cognitive models. In: K\u00fchnberger KU, Rudolph S, Wang P (eds) Artificial general intelligence. Lecture notes in computer science, vol 7999. Springer, Berlin Heidelberg, pp 130\u2013139"},{"key":"9505_CR209","unstructured":"Strickler RE (1973) Change in selected characteristics of students between ninth and twelfth grade as related to high school curriculum"},{"key":"9505_CR210","doi-asserted-by":"crossref","unstructured":"Sturtevant N (2012) Benchmarks for grid-based pathfinding. Trans Comput Intell AI Games 4(2):144\u2013148. http:\/\/web.cs.du.edu\/~sturtevant\/papers\/benchmarks.pdf","DOI":"10.1109\/TCIAIG.2012.2197681"},{"issue":"4","key":"9505_CR211","doi-asserted-by":"crossref","first-page":"337","DOI":"10.1007\/s10817-009-9143-8","volume":"43","author":"G Sutcliffe","year":"2009","unstructured":"Sutcliffe G (2009) The TPTP problem library and associated infrastructure: the FOF and CNF Parts, v3.5.0. J Autom Reason 43(4):337\u2013362","journal-title":"J Autom Reason"},{"issue":"1","key":"9505_CR212","first-page":"35","volume":"19","author":"G Sutcliffe","year":"2006","unstructured":"Sutcliffe G, Suttner C (2006) The state of CASC. AI Commun 19(1):35\u201348","journal-title":"AI Commun"},{"key":"9505_CR213","unstructured":"Thrun S (1996) Is learning the n-th thing any easier than learning the first? In: Advances in neural information processing systems, pp 640\u2013646"},{"key":"9505_CR214","volume-title":"Learning to learn","author":"S Thrun","year":"2012","unstructured":"Thrun S, Pratt L (2012) Learning to learn. Springer, New York"},{"key":"9505_CR215","doi-asserted-by":"crossref","unstructured":"Thurstone LL (1938a) Primary mental abilities. Psychometric monographs","DOI":"10.1037\/t11410-000"},{"key":"9505_CR216","doi-asserted-by":"crossref","unstructured":"Thurstone LL (1938b) Primary mental abilities. Psychometric monographs","DOI":"10.1037\/t11410-000"},{"key":"9505_CR217","unstructured":"Togelius J, Yannakakis GN, Karakovskiy S, Shaker N (2012) Assessing believability. In: Believable bots, Springer, pp 215\u2013230"},{"key":"9505_CR218","first-page":"17","volume":"3","author":"L Torrey","year":"2009","unstructured":"Torrey L, Shavlik J (2009) Transfer learning. Handb Res Mach Learn Appl 3:17\u201335","journal-title":"Handb Res Mach Learn Appl"},{"key":"9505_CR219","doi-asserted-by":"crossref","first-page":"433","DOI":"10.1093\/mind\/LIX.236.433","volume":"59","author":"AM Turing","year":"1950","unstructured":"Turing AM (1950) Computing machinery and intelligence. Mind 59:433\u2013460","journal-title":"Mind"},{"issue":"11","key":"9505_CR220","doi-asserted-by":"crossref","first-page":"1134","DOI":"10.1145\/1968.1972","volume":"27","author":"LG Valiant","year":"1984","unstructured":"Valiant LG (1984) A theory of the learnable. Commun ACM 27(11):1134\u20131142","journal-title":"Commun ACM"},{"issue":"3","key":"9505_CR221","doi-asserted-by":"crossref","first-page":"90","DOI":"10.1609\/aimag.v36i3.2571","volume":"36","author":"M Vallati","year":"2015","unstructured":"Vallati M, Chrpa L, Grzes M, McCluskey TL, Roberts M, Sanner S (2015) The 2014 international planning competition: progress and trends. AI Mag 36(3):90\u201398","journal-title":"AI Mag"},{"key":"9505_CR222","doi-asserted-by":"crossref","unstructured":"van Rijn JN, Bischl B, Torgo L, Gao B, Umaashankar V, Fischer S, Winter P, Wiswedel B, Berthold MR, Vanschoren J (2013) Openml: a collaborative science platform. In: Machine learning and knowledge discovery in databases. Springer, pp 645\u2013649","DOI":"10.1007\/978-3-642-40994-3_46"},{"issue":"2","key":"9505_CR223","doi-asserted-by":"crossref","first-page":"127","DOI":"10.1007\/s10994-011-5277-0","volume":"87","author":"J Vanschoren","year":"2012","unstructured":"Vanschoren J, Blockeel H, Pfahringer B, Holmes G (2012) Experiment databases. Mach Learn 87(2):127\u2013158","journal-title":"Mach Learn"},{"issue":"2","key":"9505_CR224","doi-asserted-by":"crossref","first-page":"49","DOI":"10.1145\/2641190.2641198","volume":"15","author":"J Vanschoren","year":"2014","unstructured":"Vanschoren J, van Rijn JN, Bischl B, Torgo L (2014) Openml: networked science in machine learning. ACM SIGKDD Explor Newsl 15(2):49\u201360","journal-title":"ACM SIGKDD Explor Newsl"},{"issue":"4","key":"9505_CR225","doi-asserted-by":"publisher","first-page":"797","DOI":"10.1109\/TPAMI.2013.163","volume":"36","author":"D V\u00e1zquez","year":"2014","unstructured":"V\u00e1zquez D, L\u00f3pez AM, Mar\u00edn J, Ponsa D, Ger\u00f3nimo D (2014) Virtual and real world adaptation for pedestrian detection. IEEE Trans Pattern Anal Mach Intell 36(4):797\u2013809. doi: 10.1109\/TPAMI.2013.163","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"issue":"03","key":"9505_CR226","doi-asserted-by":"crossref","first-page":"460","DOI":"10.1017\/S0140525X00069703","volume":"15","author":"SA Vere","year":"1992","unstructured":"Vere SA (1992) A cognitive process shell. Behav Brain Sci 15(03):460\u2013461","journal-title":"Behav Brain Sci"},{"key":"9505_CR227","doi-asserted-by":"crossref","unstructured":"von Ahn L (2009) Human computation. In: Design automation conference, 2009. DAC\u201909. 46th ACM\/IEEE, IEEE, pp 418\u2013419","DOI":"10.1145\/1629911.1630023"},{"issue":"2","key":"9505_CR228","doi-asserted-by":"crossref","first-page":"56","DOI":"10.1145\/966389.966390","volume":"47","author":"L Ahn von","year":"2004","unstructured":"von Ahn L, Blum M, Langford J (2004) Telling humans and computers apart automatically. Commun ACM 47(2):56\u201360","journal-title":"Commun ACM"},{"issue":"5895","key":"9505_CR229","doi-asserted-by":"crossref","first-page":"1465","DOI":"10.1126\/science.1160379","volume":"321","author":"L Ahn von","year":"2008","unstructured":"von Ahn L, Maurer B, McMillen C, Abraham D, Blum M (2008) RECAPTCHA: human-based character recognition via web security measures. Science 321(5895):1465","journal-title":"Science"},{"issue":"2","key":"9505_CR230","doi-asserted-by":"crossref","first-page":"185","DOI":"10.1093\/comjnl\/11.2.185","volume":"11","author":"CS Wallace","year":"1968","unstructured":"Wallace CS, Boulton DM (1968) An information measure for classification. Comput J 11(2):185\u2013194","journal-title":"Comput J"},{"issue":"4","key":"9505_CR231","doi-asserted-by":"crossref","first-page":"270","DOI":"10.1093\/comjnl\/42.4.270","volume":"42","author":"CS Wallace","year":"1999","unstructured":"Wallace CS, Dowe DL (1999) Minimum message length and Kolmogorov complexity. Comput J 42(4):270\u2013283 (special issue on Kolmogorov complexity)","journal-title":"Comput J"},{"key":"9505_CR232","unstructured":"Wang G, Mohanlal M, Wilson C, Wang X, Metzger M, Zheng H, Zhao BY (2012) Social Turing tests: crowdsourcing sybil detection. arXiv preprint arXiv:1205.3856"},{"key":"9505_CR233","doi-asserted-by":"crossref","unstructured":"Wang P (2010) The evaluation of agi systems. In: Proceedings of the third conference on artificial general intelligence, Citeseer, pp 164\u2013169","DOI":"10.2991\/agi.2010.33"},{"key":"9505_CR234","unstructured":"Warwick K (2014) Turing Test success marks milestone in computing history. University or Reading Press Release,"},{"key":"9505_CR235","volume-title":"Comparative cognition: Experimental explorations of animal intelligence","author":"EA Wasserman","year":"2006","unstructured":"Wasserman EA, Zentall TR (2006) Comparative cognition: Experimental explorations of animal intelligence. Oxford University Press, Oxford"},{"issue":"3","key":"9505_CR236","first-page":"279","volume":"8","author":"CJCH Watkins","year":"1992","unstructured":"Watkins CJCH, Dayan P (1992) Q-learning. Mach Learn 8(3):279\u2013292","journal-title":"Mach Learn"},{"issue":"1","key":"9505_CR237","doi-asserted-by":"crossref","first-page":"1","DOI":"10.2458\/v2i1.12351","volume":"2","author":"DJ Weiss","year":"2011","unstructured":"Weiss DJ (2011) Better data from better measurements using computerized adaptive testing. J Methods Meas Soc Sci 2(1):1\u201327","journal-title":"J Methods Meas Soc Sci"},{"issue":"1","key":"9505_CR238","doi-asserted-by":"crossref","first-page":"36","DOI":"10.1145\/365153.365168","volume":"9","author":"J Weizenbaum","year":"1966","unstructured":"Weizenbaum J (1966) ELIZA\u2014a computer program for the study of natural language communication between man and machine. Commun ACM 9(1):36\u201345","journal-title":"Commun ACM"},{"key":"9505_CR239","first-page":"19","volume":"21","author":"M Wellman","year":"2004","unstructured":"Wellman M, Reeves D, Lochner K, Vorobeychik Y (2004) Price prediction in a trading agent competition. J Artif Intell Res JAIR 21:19\u201336","journal-title":"J Artif Intell Res JAIR"},{"key":"9505_CR240","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1007\/s10710-012-9177-2","volume":"14","author":"DR White","year":"2013","unstructured":"White DR, McDermott J, Castelli M, Manzoni L, Goldman BW, Kronberger G, Ja\u015bkowski W, O\u2019Reilly UM, Luke S (2013) Better GP benchmarks: community survey results and proposals. Genet Program Evolvable Mach 14:3\u201329. doi: 10.1007\/s10710-012-9177-2","journal-title":"Genet Program Evolvable Mach"},{"issue":"2","key":"9505_CR241","doi-asserted-by":"crossref","first-page":"81","DOI":"10.1609\/aimag.v31i2.2227","volume":"31","author":"S Whiteson","year":"2010","unstructured":"Whiteson S, Tanner B, White A (2010) The reinforcement learning competitions. AI Mag 31(2):81\u201394","journal-title":"AI Mag"},{"key":"9505_CR242","doi-asserted-by":"crossref","unstructured":"Whiteson S, Tanner B, Taylor ME, Stone P (2011) Protecting against evaluation overfitting in empirical reinforcement learning. In: 2011 IEEE symposium on adaptive dynamic programming and reinforcement learning (ADPRL), IEEE, pp 120\u2013127","DOI":"10.1109\/ADPRL.2011.5967363"},{"key":"9505_CR243","doi-asserted-by":"crossref","unstructured":"Williams PL, Beer RD (2010) Information dynamics of evolved agents. In: From animals to animats 11, Springer, pp 38\u201349","DOI":"10.1007\/978-3-642-15193-4_4"},{"key":"9505_CR244","first-page":"71","volume":"51","author":"M Winikoff","year":"2014","unstructured":"Winikoff M, Cranefield S (2014) On the testability of bdi agent systems. J Artif Intell Res JAIR 51:71\u2013131","journal-title":"J Artif Intell Res JAIR"},{"issue":"7","key":"9505_CR245","doi-asserted-by":"crossref","first-page":"1341","DOI":"10.1162\/neco.1996.8.7.1341","volume":"8","author":"DH Wolpert","year":"1996","unstructured":"Wolpert DH (1996) The lack of a priori distinctions between learning algorithms. Neural Comput 8(7):1341\u20131390","journal-title":"Neural Comput"},{"key":"9505_CR246","unstructured":"Wolpert DH (2012) What the no free lunch theorems really mean; how to improve search algorithms. Technical report, Santa fe Institute Working Paper"},{"key":"9505_CR247","unstructured":"Wolpert DH, Macready WG (1995) No free lunch theorems for search. Technical report SFI-TR-95-02-010 (Santa Fe Institute)"},{"issue":"6","key":"9505_CR248","doi-asserted-by":"crossref","first-page":"721","DOI":"10.1109\/TEVC.2005.856205","volume":"9","author":"DH Wolpert","year":"2005","unstructured":"Wolpert DH, Macready WG (2005) Coevolutionary free lunches. IEEE Trans Evol Comput 9(6):721\u2013735","journal-title":"IEEE Trans Evol Comput"},{"key":"9505_CR249","doi-asserted-by":"crossref","DOI":"10.1201\/b18612","volume-title":"Artificial superintelligence: a futuristic approach","author":"RV Yampolskiy","year":"2015","unstructured":"Yampolskiy RV (2015) Artificial superintelligence: a futuristic approach. CRC Press, Boca Raton"},{"issue":"2","key":"9505_CR250","doi-asserted-by":"crossref","first-page":"61","DOI":"10.1177\/194675671200400210","volume":"4","author":"R Yonck","year":"2012","unstructured":"Yonck R (2012) Toward a standard metric of machine intelligence. World Future Rev 4(2):61\u201370","journal-title":"World Future Rev"},{"issue":"6218","key":"9505_CR251","doi-asserted-by":"crossref","first-page":"116","DOI":"10.1126\/science.347.6218.116","volume":"347","author":"J You","year":"2015","unstructured":"You J (2015) Beyond the turing test. Science 347(6218):116\u2013116","journal-title":"Science"},{"issue":"1","key":"9505_CR252","doi-asserted-by":"crossref","first-page":"28","DOI":"10.1177\/1059712308099230","volume":"17","author":"Z Zatuchna","year":"2009","unstructured":"Zatuchna Z, Bagnall A (2009) Learning mazes with aliasing states: an LCS algorithm with associative perception. Adapt Behav 17(1):28\u201357","journal-title":"Adapt Behav"},{"key":"9505_CR253","doi-asserted-by":"crossref","DOI":"10.1201\/b12207","volume-title":"Ensemble methods: foundations and algorithms","author":"ZH Zhou","year":"2012","unstructured":"Zhou ZH (2012) Ensemble methods: foundations and algorithms. CRC Press, Boca Raton"}],"container-title":["Artificial Intelligence Review"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10462-016-9505-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10462-016-9505-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10462-016-9505-7","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10462-016-9505-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,10]],"date-time":"2025-06-10T06:52:26Z","timestamp":1749538346000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10462-016-9505-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,8,19]]},"references-count":253,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2017,10]]}},"alternative-id":["9505"],"URL":"https:\/\/doi.org\/10.1007\/s10462-016-9505-7","relation":{},"ISSN":["0269-2821","1573-7462"],"issn-type":[{"value":"0269-2821","type":"print"},{"value":"1573-7462","type":"electronic"}],"subject":[],"published":{"date-parts":[[2016,8,19]]}}}