{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,17]],"date-time":"2026-06-17T22:41:38Z","timestamp":1781736098808,"version":"3.54.5"},"reference-count":90,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"7","license":[{"start":{"date-parts":[[2020,7,1]],"date-time":"2020-07-01T00:00:00Z","timestamp":1593561600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/501100000266","name":"U.K. Royal Academy of Engineering and through the UKRI\/EPSRC Projects","doi-asserted-by":"publisher","award":["EP\/L024845"],"award-info":[{"award-number":["EP\/L024845"]}],"id":[{"id":"10.13039\/501100000266","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000266","name":"U.K. Royal Academy of Engineering and through the UKRI\/EPSRC Projects","doi-asserted-by":"publisher","award":["EP\/R026092"],"award-info":[{"award-number":["EP\/R026092"]}],"id":[{"id":"10.13039\/501100000266","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000266","name":"U.K. Royal Academy of Engineering and through the UKRI\/EPSRC Projects","doi-asserted-by":"publisher","award":["EP\/R026173"],"award-info":[{"award-number":["EP\/R026173"]}],"id":[{"id":"10.13039\/501100000266","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000266","name":"U.K. Royal Academy of Engineering and through the UKRI\/EPSRC Projects","doi-asserted-by":"publisher","award":["EP\/R026084"],"award-info":[{"award-number":["EP\/R026084"]}],"id":[{"id":"10.13039\/501100000266","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Proc. IEEE"],"published-print":{"date-parts":[[2020,7]]},"DOI":"10.1109\/jproc.2020.2991262","type":"journal-article","created":{"date-parts":[[2020,5,18]],"date-time":"2020-05-18T21:41:57Z","timestamp":1589838117000},"page":"1011-1026","source":"Crossref","is-referenced-by-count":39,"title":["Verifiable Self-Aware Agent-Based Autonomous Systems"],"prefix":"10.1109","volume":"108","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1426-1896","authenticated-orcid":false,"given":"Louise A.","family":"Dennis","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0875-3862","authenticated-orcid":false,"given":"Michael","family":"Fisher","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.2514\/1.I010096"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-24270-0_17"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.2514\/6.2011-6664"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1007\/s10515-014-0168-9"},{"key":"ref76","first-page":"1106","article-title":"Ethical judgment of agents’ behaviors in multi-agent systems","author":"cointe","year":"2016","journal-title":"Proc 4th Int Conf Auton Agents Multiagent Syst (AAMAS)"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1016\/j.robot.2015.11.012"},{"key":"ref74","year":"2010","journal-title":"CAP 393 Air Navigation The Order and the Regulations"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/JRA.1986.1087032"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1093\/logcom\/exp029"},{"key":"ref38","year":"2020","journal-title":"ROS—Robot Operating System"},{"key":"ref78","first-page":"1933","article-title":"Recognising assumption violations in autonomous systems verification","author":"dennis","year":"2018","journal-title":"Proc 3rd Int Joint Conf Autonomous Agents and Multiagent Systems (AAMAS)"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-03769-7_15"},{"key":"ref33","first-page":"37","article-title":"Run-time evolution through explicit meta-objects","volume":"641","author":"ressia","year":"2010","journal-title":"in Proc 5th Workshop Modelsrun time"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1016\/S0004-3702(99)00047-8"},{"key":"ref31","first-page":"232","author":"winfield","year":"2014","journal-title":"Robots with Internal Models A Route to Self-Aware Hence Safer Robots"},{"key":"ref30","first-page":"77","article-title":"Robots with internal models a route to machine consciousness?","volume":"10","author":"holland","year":"2003","journal-title":"J Consciousness Stud"},{"key":"ref37","author":"duval","year":"1972","journal-title":"A Theory of Objective Self Awareness"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2003.1160055"},{"key":"ref35","doi-asserted-by":"crossref","DOI":"10.1007\/978-3-319-39675-0","author":"lewis","year":"2016","journal-title":"Self-Aware Computing Systems An Engineering Approach"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1145\/2228360.2228409"},{"key":"ref60","first-page":"995","article-title":"Actions with durations and failures in BDI languages","volume":"263","author":"dennis","year":"2014","journal-title":"Proc Europ Conf Artif Intel (ECAI)"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/4074.001.0001"},{"key":"ref61","first-page":"1","article-title":"Plan library reconfigurability in BDI agents","author":"cardoso","year":"2019","journal-title":"Proc 7th Int Workshop Eng Multi-Agent Syst (EMAS)"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1016\/S1574-6526(05)80017-3"},{"key":"ref28","first-page":"502","article-title":"A computational theory of belief introspection","author":"konolige","year":"1985","journal-title":"Proc IJCAI"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1145\/48529.48530"},{"key":"ref27","author":"johnson-laird","year":"1983","journal-title":"Mental Models Toward a Cognitive Science of Language Inference and Consciousness"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1145\/359104.359106"},{"key":"ref66","author":"boyer","year":"1981","journal-title":"The Correctness Problem in Computer Science"},{"key":"ref29","author":"maes","year":"1988","journal-title":"Meta-level Architecture and Reflection"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1023\/A:1022920129859"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1007\/s10515-011-0088-x"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-79043-3_8"},{"key":"ref2","year":"2020","journal-title":"Vacuum cleaning robotiRobot Roomba"},{"key":"ref1","year":"2020","journal-title":"Waymo Wins Industry’s First Approval to Test Driverless Cars on Public Roads in California"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/2500468.2494558"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1016\/j.tcs.2005.11.026"},{"key":"ref21","first-page":"938","article-title":"The insufficiency of formal design methods: Necessity of an experimental approach for the understanding and control of complex MAS","author":"edmonds","year":"2004","journal-title":"Proc AAMAS"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/2071356.2071363"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1016\/j.entcs.2009.09.051"},{"key":"ref26","first-page":"141","article-title":"A tutorial on runtime verification","author":"falcone","year":"2013","journal-title":"Engineering Dependable Software Systems"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1007\/s10515-005-6205-y"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1080\/09528130412331309277"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1016\/j.robot.2010.05.008"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1177\/0278364912459665"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2019.2898267"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1016\/j.cogsys.2017.04.002"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.3389\/frobt.2017.00074"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-44631-1_16"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1023\/A:1010084620690"},{"key":"ref54","first-page":"888","article-title":"Generating project networks","volume":"2","author":"tate","year":"1977","journal-title":"Proc Intern Joint Conf Artificial Intel (IJCAI)"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1016\/0004-3702(71)90010-5"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/WI-IAT.2015.58"},{"key":"ref10","author":"taylor","year":"2017","journal-title":"Autonomy"},{"key":"ref11","author":"antsaklis","year":"1993","journal-title":"An Introduction to Intelligent and Autonomous Control"},{"key":"ref40","author":"wooldridge","year":"2002","journal-title":"An Introduction to Multiagent Systems"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1017\/S0269888900008122"},{"key":"ref13","first-page":"70","article-title":"Tasking interface manager: Affording pilot control of adaptive automation and aiding","author":"bonner","year":"2004","journal-title":"Contemporary Ergonomics"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1016\/S0921-8890(98)00028-1"},{"key":"ref15","first-page":"41","article-title":"Computationally adjustable autonomy","volume":"8","author":"hexmoor","year":"2007","journal-title":"Scalable Comput Pract Exp"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-64107-2_21"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/FAS-W.2018.00054"},{"key":"ref81","article-title":"The ‘why did you do that?’ button: Answering why-questions for end users of robotic systems","author":"koeman","year":"2019","journal-title":"Proc 7th Int Workshop Eng Multi-Agent Syst (EMAS)"},{"key":"ref17","author":"clarke","year":"1999","journal-title":"Model checking"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1109\/ISSREW.2018.00028"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-31424-7_52"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-98938-9_10"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/3342355"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1145\/3103111.3104037"},{"key":"ref89","article-title":"Governing lethal behavior: Embedding ethics in a hybrid deliberative\/reactive robot architecture","author":"arkin","year":"2007"},{"key":"ref4","year":"2020","journal-title":"New Modular Care-O-Bot Generation Released Care-O-Bot 4"},{"key":"ref3","article-title":"Care-O-bot 4 is the robot servant we all want but probably can’t afford","author":"ackerman","year":"0","journal-title":"IEEE Spectr"},{"key":"ref6","year":"2020","journal-title":"Automated Trading Systems The Pros and Cons"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1142\/p930"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1109\/MIS.2010.88"},{"key":"ref8","author":"knapp","year":"2011","journal-title":"Nevada passes law authorizing driverless cars"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1016\/j.scico.2017.05.006"},{"key":"ref7","year":"2020","journal-title":"Unmanned Aircraft"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1007\/s10458-015-9293-5"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.4204\/EPTCS.257.5"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-12988-0_2"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1177\/0278364919883338"},{"key":"ref46","doi-asserted-by":"crossref","DOI":"10.7551\/mitpress\/5804.001.0001","author":"wooldridge","year":"2000","journal-title":"Reasoning About Rational Agents"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1016\/0004-3702(90)90055-5"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.21105\/joss.00617"},{"key":"ref47","article-title":"Gwendolen semantics: 2017","author":"dennis","year":"2017"},{"key":"ref42","article-title":"Reducing code complexity in hybrid control systems","author":"dennis","year":"2010","journal-title":"Proc Int Symp Artif Intell Robot Automat Space (i-SAIRAS)"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1007\/978-94-015-9204-8"},{"key":"ref44","first-page":"439","article-title":"An abstract architecture for rational agents","author":"rao","year":"1992","journal-title":"Proc Int Conf Knowl Represent Reason"},{"key":"ref43","author":"bratman","year":"1987","journal-title":"Intentions Plans and Practical Reasoning"}],"container-title":["Proceedings of the IEEE"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/5\/9120379\/09094672.pdf?arnumber=9094672","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,5]],"date-time":"2024-08-05T23:56:33Z","timestamp":1722902193000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9094672\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,7]]},"references-count":90,"journal-issue":{"issue":"7"},"URL":"https:\/\/doi.org\/10.1109\/jproc.2020.2991262","relation":{},"ISSN":["0018-9219","1558-2256"],"issn-type":[{"value":"0018-9219","type":"print"},{"value":"1558-2256","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,7]]}}}