{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,7,5]],"date-time":"2025-07-05T17:10:10Z","timestamp":1751735410342,"version":"3.41.0"},"reference-count":51,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2018,7,6]],"date-time":"2018-07-06T00:00:00Z","timestamp":1530835200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Knowl Inf Syst"],"published-print":{"date-parts":[[2019,1]]},"DOI":"10.1007\/s10115-018-1234-6","type":"journal-article","created":{"date-parts":[[2018,7,6]],"date-time":"2018-07-06T09:44:25Z","timestamp":1530870265000},"page":"35-58","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Rating the skill of synthetic agents in competitive multi-agent environments"],"prefix":"10.1007","volume":"58","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8501-8899","authenticated-orcid":false,"given":"Chairi","family":"Kiourt","sequence":"first","affiliation":[]},{"given":"Dimitris","family":"Kalles","sequence":"additional","affiliation":[]},{"given":"George","family":"Pavlidis","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2018,7,6]]},"reference":[{"key":"1234_CR1","doi-asserted-by":"crossref","unstructured":"Al-Khateeb B, Kendall G (2011) Introducing a Round Robin tournament into evolutionary individual and social learning checkers. In: Developments in E-systems engineering, Dubai, United Arab Emirates, pp 294\u2013299. https:\/\/doi.org\/110.1109\/DeSE.2011.10","DOI":"10.1109\/DeSE.2011.10"},{"issue":"1","key":"1234_CR2","first-page":"77","volume":"19","author":"S Attle","year":"2007","unstructured":"Attle S, Baker B (2007) Cooperative learning in a competitive environment: classroom applications. Int J Teach Learn High Educ 19(1):77\u201383","journal-title":"Int J Teach Learn High Educ"},{"key":"1234_CR3","unstructured":"Basili V, Caldiera G, Rombach HD (1994) Goal question metric approach. In: Encyclopedia of software engineering, pp 528\u2013532"},{"issue":"2","key":"1234_CR4","doi-asserted-by":"publisher","first-page":"156","DOI":"10.1109\/TSMCC.2007.913919","volume":"38","author":"L Busoniu","year":"2008","unstructured":"Busoniu L, Babuska R, De Schutter B (2008) A comprehensive survey of multiagent reinforcement learning. IEEE Trans Syst Man Cybern Part C Appl Rev 38(2):156\u2013172. https:\/\/doi.org\/10.1109\/TSMCC.2007.913919","journal-title":"IEEE Trans Syst Man Cybern Part C Appl Rev"},{"issue":"7","key":"1234_CR5","doi-asserted-by":"publisher","first-page":"1098","DOI":"10.1016\/j.engappai.2011.06.006","volume":"24","author":"A Caballero","year":"2011","unstructured":"Caballero A, Botia J, Gomez-Skarmeta A (2011) Using cognitive agents in social simulations. Eng Appl Artif Intell 24(7):1098\u20131109. https:\/\/doi.org\/10.1016\/j.engappai.2011.06.006","journal-title":"Eng Appl Artif Intell"},{"key":"1234_CR6","unstructured":"Calvo R (1998) Valencia Spain: the cradle of European chess. In: Presentation to the CCI, Vienna, Austria"},{"key":"1234_CR7","doi-asserted-by":"crossref","unstructured":"Di Bitonto P, Laterza M, Rosell T, Rossano V (2010) An evaluation method for multi-agent systems. In: Agent and multi-agent systems: technologies and applications, pp 32-41. https:\/\/doi.org\/110.1007\/978-3-642-13480-7_5","DOI":"10.1007\/978-3-642-13480-7_5"},{"key":"1234_CR8","doi-asserted-by":"crossref","unstructured":"Edelkamp S, Kissmann P (2008) Symbolic classication of general two-player games. In: Technical report. Technische Universitat, Dortmund","DOI":"10.3233\/978-1-58603-891-5-905"},{"key":"1234_CR9","volume-title":"The rating of chess players, past and present","author":"AE Elo","year":"1978","unstructured":"Elo AE (1978) The rating of chess players, past and present. Arco Publishing, New York"},{"key":"1234_CR10","volume-title":"Multi-agent systems: an introduction to distributed artificial intelligence","author":"J Ferber","year":"1999","unstructured":"Ferber J (1999) Multi-agent systems: an introduction to distributed artificial intelligence. Addison-Wesley, Boston"},{"key":"1234_CR11","doi-asserted-by":"crossref","unstructured":"Ferber J, Gutknecht O, Michel F (2004) From agents to organizations: an organizational view of multi-agent systems. In: In Agent-oriented software engineering IV, LNCS 2935. Springer, Berlin, pp 214\u2013230. https:\/\/doi.org\/110.1007\/978-3-540-24620-6_15","DOI":"10.1007\/978-3-540-24620-6_15"},{"key":"1234_CR12","unstructured":"Francisca T, Castro AM, Ana PR, Eugnio O (2013) Multi-agent learning in both cooperative and competitive environments. In: 16th Portuguese conference on artificial intelligence. Azores, Portugal, pp 400\u2013411"},{"issue":"2","key":"1234_CR13","doi-asserted-by":"publisher","first-page":"119","DOI":"10.1023\/A:1008811827890","volume":"9","author":"H Friedrich","year":"1998","unstructured":"Friedrich H, Rogalla O, Dillmann R (1998) Integrating skills into multi-agent systems. J Intell Manuf 9(2):119\u2013127. https:\/\/doi.org\/10.1023\/A:1008811827890","journal-title":"J Intell Manuf"},{"key":"1234_CR14","volume-title":"Simulation for the social scientist","author":"N Gilbert","year":"2005","unstructured":"Gilbert N, Troitzsch KG (2005) Simulation for the social scientist, 2nd edn. Open University Press, New York","edition":"2"},{"key":"1234_CR15","first-page":"59","volume":"3","author":"EM Glickman","year":"1995","unstructured":"Glickman EM (1995) A comprehensive guide to chess ratings. Am Chess J 3:59\u2013102","journal-title":"Am Chess J"},{"key":"1234_CR16","first-page":"377","volume":"48","author":"EM Glickman","year":"1999","unstructured":"Glickman EM (1999) Parameter estimation in large dynamic paired comparison experiments. Appl Stat 48:377\u2013394","journal-title":"Appl Stat"},{"issue":"2","key":"1234_CR17","first-page":"21","volume":"12","author":"EM Glickman","year":"1999","unstructured":"Glickman EM, Albyn JC (1999) Rating the chess rating system. Chance 12(2):21\u201328","journal-title":"Chance"},{"key":"1234_CR18","volume-title":"Official chess handbook","author":"K Harkness","year":"1967","unstructured":"Harkness K (1967) Official chess handbook. McKay, Philadelphia"},{"key":"1234_CR19","first-page":"269","volume":"19","author":"R Herbrich","year":"2007","unstructured":"Herbrich R, Minka T, Graepel T (2007) TrueSkill(TM): a Bayesian skill rating system. Adv Neural Inf Process Syst 19:269\u2013576","journal-title":"Adv Neural Inf Process Syst"},{"key":"1234_CR20","doi-asserted-by":"crossref","unstructured":"Hoen JP, Tuyls K, Panait L, Luke S, La Poutr JA (2005) An overview of cooperative and competitive multiagent learning. In: Proceedings of the first international conference on learning and adaption in multi-agent systems, Utrecht, The Netherlands, pp 1\u201346. https:\/\/doi.org\/10.1007\/11691839_1","DOI":"10.1007\/11691839_1"},{"key":"1234_CR21","unstructured":"Kalles D (2007) Measuring expert impact on learning how to play a board game. In: IFIP conference on artificial intelligence applications and innovations, Athens, Greece"},{"key":"1234_CR22","doi-asserted-by":"crossref","unstructured":"Kalles D, Kanellopoulos P (2001) On verifying game design and playing strategies using reinforcement learning. In: Proceedings of ACM symposium on applied computing, special track on artificial intelligence and computation logic. ACM, Las Vegas, pp 6\u201311","DOI":"10.1145\/372202.372204"},{"key":"1234_CR23","unstructured":"Kapetanakis S, Kudenko D (2002) Reinforcement learning of coordination in cooperative multi-agent system. In: Proceeding of eighteenth national conference on artificial intelligence. Edmonton, Alberta, pp 326\u2013331"},{"issue":"1\/2","key":"1234_CR24","doi-asserted-by":"publisher","first-page":"81","DOI":"10.2307\/2332226","volume":"30","author":"M Kendall","year":"1936","unstructured":"Kendall M (1936) A new measure of rank correlation. Biometrica 30(1\/2):81\u201393","journal-title":"Biometrica"},{"key":"1234_CR25","doi-asserted-by":"crossref","unstructured":"Kiourt C, Kalles D (2012) Social reinforcement learning in game playing. In: IEEE international conference on tools with artificial intelligence, Athens, Greece, pp 322\u2013326. https:\/\/doi.org\/10.1109\/ICTAI.2012.51","DOI":"10.1109\/ICTAI.2012.51"},{"issue":"6","key":"1234_CR26","doi-asserted-by":"publisher","first-page":"411","DOI":"10.1177\/1059712316679239","volume":"24","author":"C Kiourt","year":"2016","unstructured":"Kiourt C, Kalles D (2016) Synthetic learning agents in game-playing social environments. Adapt Behav 24(6):411\u2013427. https:\/\/doi.org\/10.1177\/1059712316679239","journal-title":"Adapt Behav"},{"key":"1234_CR27","doi-asserted-by":"crossref","unstructured":"Kiourt C, Pavlidis G, Kalles D (2016a) Human rating methods on multi-agent systems. In: Multi-agent systems and agreement technologies: 13th European conference, EUMAS 2015, and third international conference, AT 2015, Athens, Greece. Springer, pp 129\u2013136. https:\/\/doi.org\/10.1007\/978-3-319-33509-4_11","DOI":"10.1007\/978-3-319-33509-4_11"},{"key":"1234_CR28","doi-asserted-by":"crossref","unstructured":"Kiourt C, Pavlidis G, Kalles D (2016b) ReSkill: relative skill-level calculation system. In: 9th hellenic conference on artificial intelligence (SETN2016), Thessaloniki, Greece, p 39. https:\/\/doi.org\/10.1145\/2903220.2903224","DOI":"10.1145\/2903220.2903224"},{"key":"1234_CR29","doi-asserted-by":"crossref","unstructured":"Kiourt C, Kalles D and Kanellopoulos P (2017) How game complexity affects the playing behavior of synthetic agents. In: 15th European conference on multi-agent systems (EUMAS2017), France","DOI":"10.1007\/978-3-030-01713-2_22"},{"issue":"1","key":"1234_CR30","doi-asserted-by":"publisher","first-page":"198","DOI":"10.1109\/TASE.2011.2163510","volume":"9","author":"Y Kotb","year":"2012","unstructured":"Kotb Y, Beauchemin S, Barron J (2012) Workflow nets for multiagent cooperation. IEEE Trans Autom Sci Eng 9(1):198\u2013203. https:\/\/doi.org\/10.1109\/TASE.2011.2163510","journal-title":"IEEE Trans Autom Sci Eng"},{"key":"1234_CR31","doi-asserted-by":"publisher","DOI":"10.1515\/9781400841677","volume-title":"Who\u2019s #1?: The science of rating and ranking","author":"NA Langville","year":"2012","unstructured":"Langville NA, Meyer DC (2012) Who\u2019s #1?: The science of rating and ranking. Princeton University Press, Princeton"},{"key":"1234_CR32","doi-asserted-by":"crossref","unstructured":"Liang, Shi W (2005) Performance evaluation of rating aggregation algorithms in reputation systems. In: International conference on collaborative computing: networking, applications and worksharing, p 10. https:\/\/doi.org\/10.1109\/COLCOM.2005.1651235","DOI":"10.1109\/COLCOM.2005.1651235"},{"key":"1234_CR33","unstructured":"Logan Y, Kagan T (2013) Elo ratings for structural credit assignment in multiagent systems. In: Twenty-seventh AAAI conference on artificial intelligence (late-breaking developments)"},{"issue":"1","key":"1234_CR34","doi-asserted-by":"publisher","first-page":"71","DOI":"10.1287\/orsc.2.1.71","volume":"2","author":"JG March","year":"1991","unstructured":"March JG (1991) Exploration and exploitation in organizational learning. Organ Sci 2(1):71\u201387","journal-title":"Organ Sci"},{"key":"1234_CR35","doi-asserted-by":"crossref","unstructured":"Marivate VN, Marwala T (2008) Social learning methods in board game agents. In: IEEE symposium computational intelligence and games, Perth, Australia, pp 323\u2013328. https:\/\/doi.org\/10.1109\/CIG.2008.5035657","DOI":"10.1109\/CIG.2008.5035657"},{"key":"1234_CR36","unstructured":"Nikolenko S, Sirotkin A (2011) A new bayesian rating system for team competitions. In: Proceedings of the 28th international conference on machine learning, Bellevue, Washington, pp 601\u2013608"},{"issue":"3","key":"1234_CR37","doi-asserted-by":"publisher","first-page":"387","DOI":"10.1007\/s10458-005-2631-2","volume":"11","author":"L Panait","year":"2005","unstructured":"Panait L, Luke S (2005) Cooperative multi-agent learning: the state of the art. Auton Agent Multi-Agent Syst 11(3):387\u2013434. https:\/\/doi.org\/10.1007\/s10458-005-2631-2","journal-title":"Auton Agent Multi-Agent Syst"},{"key":"1234_CR38","doi-asserted-by":"crossref","unstructured":"Pieter JH, Tuyls K, Panait L, Luke S (2006) An overview of cooperative and competitive multiagent learning. In: Learning and adaption in multi-agent systems. Lecture notes in computer science, vol 3898, pp 1\u201346. https:\/\/doi.org\/10.1007\/11691839_1","DOI":"10.1007\/11691839_1"},{"key":"1234_CR39","doi-asserted-by":"crossref","unstructured":"Ruohomaa S, Kutvonen L, Koutrouli E (2007) Reputation management survey. In: ARES\u201907: the second international conference on availability, reliability and security, pp 103\u2013111. https:\/\/doi.org\/10.1109\/ARES.2007.123","DOI":"10.1109\/ARES.2007.123"},{"key":"1234_CR40","volume-title":"Artificial intelligence: a modern approach","author":"S Russell","year":"2010","unstructured":"Russell S, Norving P (2010) Artificial intelligence: a modern approach. Prentice-Hall Inc, Upper Saddle River"},{"key":"1234_CR41","volume-title":"Multiagent systems: algorithmic, game-theoretic and logical foundations","author":"Y Shoham","year":"2009","unstructured":"Shoham Y, Leyton KB (2009) Multiagent systems: algorithmic, game-theoretic and logical foundations. Cambridge University Press, New York"},{"issue":"1","key":"1234_CR42","doi-asserted-by":"publisher","first-page":"72","DOI":"10.2307\/1412159","volume":"15","author":"C Spearman","year":"1904","unstructured":"Spearman C (1904) The proof and measurement of association between two things. Am J Psychol 15(1):72\u2013101","journal-title":"Am J Psychol"},{"issue":"1","key":"1234_CR43","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1023\/A:1022633531479","volume":"3","author":"R Sutton","year":"1988","unstructured":"Sutton R (1988) Learning to predict by the methods of temporal differences. Mach Learn 3(1):9\u201344. https:\/\/doi.org\/10.1023\/A:1022633531479","journal-title":"Mach Learn"},{"key":"1234_CR44","doi-asserted-by":"publisher","DOI":"10.1109\/TNN.1998.712192","volume-title":"Reinforcement learning\u2014an introduction","author":"R Sutton","year":"1998","unstructured":"Sutton R, Barto A (1998) Reinforcement learning\u2014an introduction. MIT Press, Cambridge"},{"issue":"4","key":"1234_CR45","doi-asserted-by":"publisher","first-page":"e0172395","DOI":"10.1371\/journal.pone.0172395","volume":"12","author":"A Tampuu","year":"2017","unstructured":"Tampuu A, Matiisen T, Kodelja D, Kuzovkin I, Korjus K, Aru J, Vicente R (2017) Multiagent cooperation and competition with deep reinforcement learning. PLoS ONE 12(4):e0172395. https:\/\/doi.org\/10.1371\/journal.pone.0172395","journal-title":"PLoS ONE"},{"key":"1234_CR46","doi-asserted-by":"publisher","first-page":"257","DOI":"10.1007\/BF00992697","volume":"4","author":"G Tesauro","year":"1992","unstructured":"Tesauro G (1992) Practical issues in temporal difference learning. Mach Learn 4:257\u2013277. https:\/\/doi.org\/10.1007\/BF00992697","journal-title":"Mach Learn"},{"issue":"3","key":"1234_CR47","doi-asserted-by":"publisher","first-page":"56","DOI":"10.1145\/203330.203343","volume":"38","author":"G Tesauro","year":"1995","unstructured":"Tesauro G (1995) Temporal difference learning and TD-gammon. Commun ACM 38(3):56\u201368. https:\/\/doi.org\/10.1145\/203330.203343","journal-title":"Commun ACM"},{"key":"1234_CR48","doi-asserted-by":"publisher","first-page":"181","DOI":"10.1016\/S0004-3702(01)00110-2","volume":"134","author":"G Tesauro","year":"2002","unstructured":"Tesauro G (2002) Programming backgammon using self-teaching neural nets. Artif Intell 134:181\u2013199. https:\/\/doi.org\/10.1016\/S0004-3702(01)00110-2","journal-title":"Artif Intell"},{"issue":"1","key":"1234_CR49","doi-asserted-by":"publisher","first-page":"110","DOI":"10.3233\/ICG-2008-31205","volume":"31","author":"J Tromp","year":"2008","unstructured":"Tromp J (2008) Solving connect-4 on medium board sizes. ICGA J 31(1):110\u2013112","journal-title":"ICGA J"},{"key":"1234_CR50","first-page":"267","volume":"12","author":"RC Weng","year":"2011","unstructured":"Weng RC, Lin C (2011) A bayesian approximation method for online ranking. J Mach Learn Res 12:267\u2013300","journal-title":"J Mach Learn Res"},{"key":"1234_CR51","unstructured":"Wilensky U (1999) NetLogo itself. In: Center for connected learning and computer-based modeling. Northwestern University, Evanston"}],"container-title":["Knowledge and Information Systems"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10115-018-1234-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-018-1234-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-018-1234-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,5]],"date-time":"2025-07-05T16:28:22Z","timestamp":1751732902000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10115-018-1234-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,7,6]]},"references-count":51,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2019,1]]}},"alternative-id":["1234"],"URL":"https:\/\/doi.org\/10.1007\/s10115-018-1234-6","relation":{},"ISSN":["0219-1377","0219-3116"],"issn-type":[{"type":"print","value":"0219-1377"},{"type":"electronic","value":"0219-3116"}],"subject":[],"published":{"date-parts":[[2018,7,6]]},"assertion":[{"value":"27 April 2017","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"2 May 2018","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"26 May 2018","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"6 July 2018","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}