{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,12]],"date-time":"2026-05-12T05:18:32Z","timestamp":1778563112535,"version":"3.51.4"},"reference-count":46,"publisher":"Springer Science and Business Media LLC","issue":"32","license":[{"start":{"date-parts":[[2021,9,4]],"date-time":"2021-09-04T00:00:00Z","timestamp":1630713600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2021,9,4]],"date-time":"2021-09-04T00:00:00Z","timestamp":1630713600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Neural Comput & Applic"],"published-print":{"date-parts":[[2023,11]]},"DOI":"10.1007\/s00521-021-06466-w","type":"journal-article","created":{"date-parts":[[2021,9,4]],"date-time":"2021-09-04T19:22:40Z","timestamp":1630783360000},"page":"23411-23428","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":19,"title":["Persistent rule-based interactive reinforcement learning"],"prefix":"10.1007","volume":"35","author":[{"given":"Adam","family":"Bignold","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1131-3382","authenticated-orcid":false,"given":"Francisco","family":"Cruz","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Richard","family":"Dazeley","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Peter","family":"Vamplew","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Cameron","family":"Foale","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2021,9,4]]},"reference":[{"key":"6466_CR1","doi-asserted-by":"crossref","unstructured":"Arzate C, Igarashi T (2020) A survey on interactive reinforcement learning: design principles and open challenges. In: Proceedings of the 2020 ACM designing interactive systems conference. pp 1195\u20131209","DOI":"10.1145\/3357236.3395525"},{"key":"6466_CR2","doi-asserted-by":"publisher","first-page":"120757","DOI":"10.1109\/ACCESS.2020.3006254","volume":"8","author":"J Lin","year":"2020","unstructured":"Lin J, Ma Z, Gomez R, Nakamura K, He B, Li G (2020) A review on interactive reinforcement learning from human social feedback. IEEE Access 8:120757\u2013120765","journal-title":"IEEE Access"},{"key":"6466_CR3","doi-asserted-by":"crossref","unstructured":"Bignold A, Cruz F, Dazeley R, Vamplew P, Foale C (2020) Human engagement providing evaluative and informative advice for interactive reinforcement learning arXiv preprint arXiv:2009.09575","DOI":"10.1007\/s00521-021-06466-w"},{"key":"6466_CR4","doi-asserted-by":"crossref","unstructured":"Knox WB, Stone P (2009) Interactively shaping agents via human reinforcement: The TAMER framework. In: Proceedings of the fifth international conference on knowledge capture, pp.\u00a09\u201316, ACM","DOI":"10.1145\/1597735.1597738"},{"key":"6466_CR5","doi-asserted-by":"crossref","unstructured":"Bignold A, Cruz F, Taylor ME, Brys T, Dazeley R, Vamplew P, Foale C (2020) A conceptual framework for externally-influenced agents: an assisted reinforcement learning review, arXiv preprint arXiv:2007.01544","DOI":"10.1007\/s12652-021-03489-y"},{"key":"6466_CR6","unstructured":"Griffith S, Subramanian K, Scholz J, Isbell C, Thomaz AL (2013) Policy shaping: integrating human feedback with reinforcement learning. In: Advances in neural information processing systems. pp 2625\u20132633"},{"key":"6466_CR7","unstructured":"Knox WB, and Stone P (2010) Combining manual feedback with subsequent MDP reward signals for reinforcement learning. In: Proceedings of the 9th international conference on autonomous agents and multiagent systems, vol 1, pp\u00a05\u201312"},{"issue":"1","key":"6466_CR8","doi-asserted-by":"publisher","first-page":"45","DOI":"10.1080\/09540091.2014.885279","volume":"26","author":"ME Taylor","year":"2014","unstructured":"Taylor ME, Carboni N, Fachantidis A, Vlahavas I, Torrey L (2014) Reinforcement learning agents providing advice in complex video games. Connect Sci 26(1):45\u201363","journal-title":"Connect Sci"},{"key":"6466_CR9","volume-title":"Reinforcement learning: an introduction","author":"RS Sutton","year":"2018","unstructured":"Sutton RS, Barto AG (2018) Reinforcement learning: an introduction. MIT Press, Cambridge"},{"key":"6466_CR10","doi-asserted-by":"publisher","DOI":"10.1002\/9780470316887","volume-title":"Markov decision processes: discrete stochastic dynamic programming","author":"ML Puterman","year":"1994","unstructured":"Puterman ML (1994) Markov decision processes: discrete stochastic dynamic programming. Wiley, Hoboken"},{"key":"6466_CR11","doi-asserted-by":"crossref","unstructured":"Sledge IJ, Pr\u00edncipe JC (2017) Balancing exploration and exploitation in reinforcement learning using a value of information criterion. In: 2017 IEEE international conference on acoustics, speech and signal processing (ICASSP), pp.\u00a02816\u20132820","DOI":"10.1109\/ICASSP.2017.7952670"},{"key":"6466_CR12","unstructured":"Subramanian K, Isbell CL Jr, Thomaz AL (2016) Exploration from demonstration for interactive reinforcement learning. In: Proceedings of the 2016 international conference on autonomous agents & multiagent systems, pp\u00a0447\u2013456"},{"issue":"16","key":"6466_CR13","doi-asserted-by":"publisher","first-page":"5574","DOI":"10.3390\/app10165574","volume":"10","author":"I Moreira","year":"2020","unstructured":"Moreira I, Rivas J, Cruz F, Dazeley R, Ayala A, Fernandes B (2020) Deep reinforcement learning with interactive feedback in a human-robot environment. Appl Sci 10(16):5574","journal-title":"Appl Sci"},{"key":"6466_CR14","unstructured":"Thomaz AL, Hoffman G, Breazeal C (2005) Real-time interactive reinforcement learning for robots. In: AAAI 2005 workshop on human comprehensible machine learning"},{"key":"6466_CR15","doi-asserted-by":"crossref","unstructured":"Ayala A, Henr\u00edquez C, Cruz F (2019) Reinforcement learning using continuous states and interactive feedback. In: Proceedings of the international conference on applications of intelligent systems, pp\u00a01\u20135","DOI":"10.1145\/3309772.3309801"},{"key":"6466_CR16","unstructured":"Mill\u00e1n C, Fernandes B, Cruz F (2019) Human feedback in continuous actor-critic reinforcement learning. In: Proceedings of the European symposium on artificial neural networks, computational intelligence and machine learning ESANN, pp\u00a0661\u2013666, ESANN"},{"key":"6466_CR17","unstructured":"Pilarski PM, and Sutton RS (2012) Between instruction and reward: human-prompted switching. In: AAAI fall symposium series: robots learning interactively from human teachers, pp\u00a045\u201352"},{"key":"6466_CR18","doi-asserted-by":"crossref","unstructured":"Cruz F, W\u00fcppen P, Magg S, Fazrie A, Wermter S (2017) Agent-advising approaches in an interactive reinforcement learning scenario. In: Proceedings of the joint IEEE international conference on development and learning and epigenetic robotics ICDL-EpiRob, pp\u00a0209\u2013214, IEEE","DOI":"10.1109\/DEVLRN.2017.8329809"},{"key":"6466_CR19","unstructured":"Torrey L, Taylor ME (2013) Teaching on a budget: agents advising agents in reinforcement learning, In: Proceedings of the international conference on autonomous agents and multiagent systems AAMAS"},{"key":"6466_CR20","doi-asserted-by":"crossref","unstructured":"L\u00f3pez G, Quesada L, Guerrero LA (2017) Alexa vs. siri vs. cortana vs. google assistant: a comparison of speech-based natural user interfaces. In: International conference on applied human factors and ergonomics, pp\u00a0241\u2013250, Springer","DOI":"10.1007\/978-3-319-60366-7_23"},{"key":"6466_CR21","unstructured":"Churamani N, Cruz F, Griffiths S, and Barros P (2016) iCub: learning emotion expressions using human reward. In: Proceedings of the workshop on bio-inspired social robot learning in home scenarios. IEEE\/RSJ IROS, p\u00a02"},{"key":"6466_CR22","doi-asserted-by":"crossref","unstructured":"Kwok SW, Carter C (1990) Multiple decision trees. In: Machine intelligence and pattern recognition, vol 9. Elsevier, pp 327\u2013335","DOI":"10.1016\/B978-0-444-88650-7.50030-5"},{"key":"6466_CR23","doi-asserted-by":"crossref","unstructured":"Rokach L, Maimon O (2005) Decision trees, in data mining and knowledge discovery handbook. Springer, pp 165\u2013192","DOI":"10.1007\/0-387-25465-X_9"},{"issue":"1","key":"6466_CR24","doi-asserted-by":"publisher","first-page":"81","DOI":"10.1007\/BF00116251","volume":"1","author":"JR Quinlan","year":"1986","unstructured":"Quinlan JR (1986) Induction of decision trees. Mach Learn 1(1):81\u2013106","journal-title":"Mach Learn"},{"key":"6466_CR25","doi-asserted-by":"publisher","DOI":"10.1201\/9781315139470","volume-title":"Classification and regression trees","author":"L Breiman","year":"2017","unstructured":"Breiman L (2017) Classification and regression trees. Routledge, Milton Park"},{"issue":"1","key":"6466_CR26","doi-asserted-by":"publisher","first-page":"7","DOI":"10.1023\/A:1007694015589","volume":"43","author":"S D\u017eeroski","year":"2001","unstructured":"D\u017eeroski S, De Raedt L, Driessens K (2001) Relational reinforcement learning. Mach Learn 43(1):7\u201352","journal-title":"Mach Learn"},{"key":"6466_CR27","doi-asserted-by":"crossref","unstructured":"Li R, Jabri A, Darrell T, Agrawal P (2020) Towards practical multi-object manipulation using relational reinforcement learning. In: IEEE international conference on robotics and automation, pp\u00a04051\u20134058","DOI":"10.1109\/ICRA40945.2020.9197468"},{"key":"6466_CR28","unstructured":"Tadepalli P, Givan R, Driessens K (2004) Relational reinforcement learning: an overview. In: Proceedings of the ICML-2004 workshop on relational reinforcement learning, pp 1\u20139"},{"key":"6466_CR29","doi-asserted-by":"publisher","first-page":"113420","DOI":"10.1016\/j.eswa.2020.113420","volume":"156","author":"R Glatt","year":"2020","unstructured":"Glatt R, Da Silva FL, da Costa Bianchi RA, Costa AHR (2020) DECAF: deep case-based policy inference for knowledge transfer in reinforcement learning. Expert Syst Appl 156:113420","journal-title":"Expert Syst Appl"},{"key":"6466_CR30","doi-asserted-by":"crossref","unstructured":"Bianchi RA, Ros R, De Mantaras RL (2009) Improving reinforcement learning by using case based heuristics. In: International conference on case-based reasoning. Springer, pp 75\u201389","DOI":"10.1007\/978-3-642-02998-1_7"},{"issue":"7","key":"6466_CR31","first-page":"1633","volume":"10","author":"ME Taylor","year":"2009","unstructured":"Taylor ME, Stone P (2009) Transfer learning for reinforcement learning domains: a survey. J Mach Learn Res 10(7):1633\u20131685","journal-title":"J Mach Learn Res"},{"key":"6466_CR32","doi-asserted-by":"publisher","first-page":"102","DOI":"10.1016\/j.artint.2015.05.008","volume":"226","author":"RA Bianchi","year":"2015","unstructured":"Bianchi RA, Celiberto LA Jr, Santos PE, Matsuura JP, de Mantaras RL (2015) Transferring knowledge as heuristics in reinforcement learning: a case-based approach. Artif Intell 226:102\u2013121","journal-title":"Artif Intell"},{"key":"6466_CR33","unstructured":"Kang B, Compton P, and Preston P (1995) Multiple classification ripple down rules: evaluation and possibilities. In: Proceedings 9th Banff knowledge acquisition for knowledge-based systems workshop, vol\u00a01, pp\u00a017\u20131"},{"key":"6466_CR34","unstructured":"Compton P, Edwards G, Kang B, Lazarus L, Malor R, Menzies T, Preston P, Srinivasan A, Sammut C (1991) Ripple down rules: possibilities and limitations. In: Proceedings of the sixth AAAI knowledge acquisition for knowledge-based systems workshop. University of Calgary, Calgary, Canada, pp 6\u20131"},{"key":"6466_CR35","doi-asserted-by":"publisher","first-page":"342","DOI":"10.1016\/j.eswa.2018.06.049","volume":"112","author":"D Herbert","year":"2018","unstructured":"Herbert D, Kang BH (2018) Intelligent conversation system using multiple classification ripple down rules and conversational context. Expert Syst Appl 112:342\u2013352","journal-title":"Expert Syst Appl"},{"issue":"2","key":"6466_CR36","doi-asserted-by":"publisher","first-page":"159","DOI":"10.1017\/S0269888909000241","volume":"24","author":"D Richards","year":"2009","unstructured":"Richards D (2009) Two decades of ripple down rules research. Knowl Eng Rev 24(2):159\u2013184","journal-title":"Knowl Eng Rev"},{"key":"6466_CR37","unstructured":"Randl\u00f8v J and Alstr\u00f8m P (1988) Learning to drive a bicycle using reinforcement learning and shaping. In: ICML, vol\u00a098, pp\u00a0463\u2013471, Citeseer"},{"key":"6466_CR38","first-page":"278","volume":"99","author":"AY Ng","year":"1999","unstructured":"Ng AY, Harada D, Russell S (1999) Policy invariance under reward transformations: theory and application to reward shaping. Proc. Int. Conf. Mach. Learn. ICML 99:278\u2013287","journal-title":"Proc. Int. Conf. Mach. Learn. ICML"},{"key":"6466_CR39","unstructured":"Devlin S and Kudenko D (2011) Theoretical considerations of potential-based reward shaping for multi-agent systems. In: The 10th international conference on autonomous agents and multiagent systems-vol 1, pp\u00a0225\u2013232"},{"key":"6466_CR40","doi-asserted-by":"crossref","unstructured":"Harutyunyan A, Devlin S, Vrancxn P, Now\u00e9 A (2015) Expressing arbitrary reward functions as potential-based advice.. In: AAAI, pp\u00a02652\u20132658","DOI":"10.1609\/aaai.v29i1.9628"},{"key":"6466_CR41","doi-asserted-by":"crossref","unstructured":"Fern\u00e1ndez F, Veloso M (2006) Probabilistic policy reuse in a reinforcement learning agent, in Proceedings of the fifth International Joint Conference on Autonomous Agents and Multi-Agent Systems. pp 720\u2013727","DOI":"10.1145\/1160633.1160762"},{"issue":"1","key":"6466_CR42","doi-asserted-by":"publisher","first-page":"13","DOI":"10.3390\/biomimetics6010013","volume":"6","author":"A Bignold","year":"2021","unstructured":"Bignold A, Cruz F, Dazeley R, Vamplew P, Foale C (2021) An evaluation methodology for interactive reinforcement learning with simulated users. Biomimetics 6(1):13","journal-title":"Biomimetics"},{"key":"6466_CR43","unstructured":"Kang BH, Preston P, Compton P, (1998) Simulated expert evaluation of multiple classification ripple down rules. In: Proceedings of the 11th workshop on knowledge acquisition, modeling and management"},{"key":"6466_CR44","volume-title":"The use of simulated experts in evaluating knowledge acquisition","author":"P Compton","year":"1995","unstructured":"Compton P, Preston P, Kang B (1995) The use of simulated experts in evaluating knowledge acquisition. University of Calgary, Calgary"},{"issue":"3","key":"6466_CR45","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1007\/BF00962234","volume":"5","author":"BR Gaines","year":"1995","unstructured":"Gaines BR, Compton P (1995) Induction of ripple-down rules applied to modeling large databases. J Intell Inf Syst 5(3):211\u2013228","journal-title":"J Intell Inf Syst"},{"key":"6466_CR46","doi-asserted-by":"crossref","unstructured":"Compton P, Peters L, Edwards G, Lavers TG (2006) Experience with ripple-down rules. Applications and innovations in intelligent systems XIII. Springer, pp 109\u2013121","DOI":"10.1007\/1-84628-224-1_9"}],"container-title":["Neural Computing and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-021-06466-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00521-021-06466-w\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-021-06466-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,10,17]],"date-time":"2023-10-17T18:16:52Z","timestamp":1697566612000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00521-021-06466-w"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,9,4]]},"references-count":46,"journal-issue":{"issue":"32","published-print":{"date-parts":[[2023,11]]}},"alternative-id":["6466"],"URL":"https:\/\/doi.org\/10.1007\/s00521-021-06466-w","relation":{},"ISSN":["0941-0643","1433-3058"],"issn-type":[{"value":"0941-0643","type":"print"},{"value":"1433-3058","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,9,4]]},"assertion":[{"value":"2 February 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"26 August 2021","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"4 September 2021","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflicts of interest"}}]}}