{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,5]],"date-time":"2026-01-05T22:23:34Z","timestamp":1767651814444,"version":"3.37.3"},"reference-count":35,"publisher":"Springer Science and Business Media LLC","issue":"8","license":[{"start":{"date-parts":[[2024,4,1]],"date-time":"2024-04-01T00:00:00Z","timestamp":1711929600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,4,1]],"date-time":"2024-04-01T00:00:00Z","timestamp":1711929600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/100016073","name":"Key Technologies Research and Development Program of Anhui Province","doi-asserted-by":"publisher","award":["2022YFB3104500"],"award-info":[{"award-number":["2022YFB3104500"]}],"id":[{"id":"10.13039\/100016073","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["U20A20177,U22B2022"],"award-info":[{"award-number":["U20A20177,U22B2022"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100013068","name":"Key Technologies Research and Development Program of Guangzhou Municipality","doi-asserted-by":"publisher","award":["2021BAA025"],"award-info":[{"award-number":["2021BAA025"]}],"id":[{"id":"10.13039\/501100013068","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Industry-University-Research Innovation Fund for Chinese Universities","award":["2021FNA04004"],"award-info":[{"award-number":["2021FNA04004"]}]},{"DOI":"10.13039\/100020738","name":"Guangdong Key Laboratory of Solid Waste Pollution Control and Recycling","doi-asserted-by":"publisher","award":["GML-KF-22-07"],"award-info":[{"award-number":["GML-KF-22-07"]}],"id":[{"id":"10.13039\/100020738","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Appl Intell"],"published-print":{"date-parts":[[2024,4]]},"DOI":"10.1007\/s10489-024-05464-4","type":"journal-article","created":{"date-parts":[[2024,5,3]],"date-time":"2024-05-03T08:02:16Z","timestamp":1714723336000},"page":"5976-5995","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["CAAC: An effective reinforcement learning algorithm for sparse reward in automatic control systems"],"prefix":"10.1007","volume":"54","author":[{"given":"Kun","family":"Liu","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Libing","family":"Wu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7749-2718","authenticated-orcid":false,"given":"Zhuangzhuang","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xinrong","family":"Hu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Na","family":"Lu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xuejiang","family":"Wei","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,5,3]]},"reference":[{"issue":"10","key":"5464_CR1","doi-asserted-by":"publisher","first-page":"11654","DOI":"10.1109\/TPAMI.2023.3285634","volume":"45","author":"X Li","year":"2023","unstructured":"Li X, Lei H, Zhang L, Wang M (2023) Differentiable logic policy for interpretable deep reinforcement learning: A study from an optimization perspective. IEEE Trans Pattern Anal Mach Intell 45(10):11654\u201311667","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"issue":"6","key":"5464_CR2","doi-asserted-by":"publisher","first-page":"4188","DOI":"10.1109\/TII.2020.3006199","volume":"17","author":"Z Xia","year":"2021","unstructured":"Xia Z, Xue S, Wu J, Chen Y, Chen J, Wu L (2021) Deep reinforcement learning for smart city communication networks. IEEE Trans Industr Inf 17(6):4188\u20134196. https:\/\/doi.org\/10.1109\/TII.2020.3006199","journal-title":"IEEE Trans Industr Inf"},{"issue":"14","key":"5464_CR3","doi-asserted-by":"publisher","first-page":"11427","DOI":"10.1109\/JIOT.2021.3054649","volume":"8","author":"Y Xu","year":"2021","unstructured":"Xu Y, Zhou H, Ma T, Zhao J, Qian B, Shen X (2021) Leveraging multiagent learning for automated vehicles scheduling at nonsignalized intersections. IEEE Internet Things J 8(14):11427\u201311439. https:\/\/doi.org\/10.1109\/JIOT.2021.3054649","journal-title":"IEEE Internet Things J"},{"issue":"12","key":"5464_CR4","doi-asserted-by":"publisher","first-page":"14903","DOI":"10.1007\/S10489-022-04227-3","volume":"53","author":"L G\u00fcitta-L\u00f3pez","year":"2023","unstructured":"G\u00fcitta-L\u00f3pez L, Boal J, L\u00f3pez-L\u00f3pez \u00c1J (2023) Learning more with the same effort: how randomization improves the robustness of a robotic deep reinforcement learning agent. Appl Intell 53(12):14903\u201314917. https:\/\/doi.org\/10.1007\/S10489-022-04227-3","journal-title":"Appl Intell"},{"issue":"7","key":"5464_CR5","doi-asserted-by":"publisher","first-page":"6774","DOI":"10.1109\/TITS.2021.3062072","volume":"23","author":"M Wang","year":"2022","unstructured":"Wang M, Wu L, Li J, He L (2022) Traffic signal control with reinforcement learning based on region-aware cooperative strategy. IEEE Trans Intell Transp Syst 23(7):6774\u20136785. https:\/\/doi.org\/10.1109\/TITS.2021.3062072","journal-title":"IEEE Trans Intell Transp Syst"},{"issue":"7676","key":"5464_CR6","doi-asserted-by":"publisher","first-page":"354","DOI":"10.1038\/NATURE24270","volume":"550","author":"D Silver","year":"2017","unstructured":"Silver D, Schrittwieser J, Simonyan K, Antonoglou I, Huang A, Guez A, Hubert T, Baker L, Lai M, Bolton A et al (2017) Mastering the game of go without human knowledge. Nature 550(7676):354\u2013359. https:\/\/doi.org\/10.1038\/NATURE24270","journal-title":"Nature"},{"issue":"7782","key":"5464_CR7","doi-asserted-by":"publisher","first-page":"350","DOI":"10.1038\/S41586-019-1724-Z","volume":"575","author":"O Vinyals","year":"2019","unstructured":"Vinyals O, Babuschkin I, Czarnecki WM, Mathieu M, Dudzik A, Chung J, Choi DH, Powell R, Ewalds T, Georgiev P et al (2019) Grandmaster level in starcraft II using multi-agent reinforcement learning. Nature 575(7782):350\u2013354. https:\/\/doi.org\/10.1038\/S41586-019-1724-Z","journal-title":"Nature"},{"unstructured":"Agarwal R, Schwarzer M, Castro PS, Courville AC, Bellemare M (2022) Reincarnating reinforcement learning: Reusing prior computation to accelerate progress. In: Advances in neural information processing systems, vol 35, pp 28955\u201328971. http:\/\/papers.nips.cc\/paper_files\/paper\/2022\/hash\/ba1c5356d9164bb64c446a4b690226b0-Abstract-Conference.html","key":"5464_CR8"},{"issue":"2","key":"5464_CR9","doi-asserted-by":"publisher","first-page":"737","DOI":"10.1109\/TAC.2022.3145632","volume":"68","author":"FA Yaghmaie","year":"2022","unstructured":"Yaghmaie FA, Gustafsson F, Ljung L (2022) Linear quadratic control using model-free reinforcement learning. IEEE Trans Autom Control 68(2):737\u2013752","journal-title":"IEEE Trans Autom Control"},{"doi-asserted-by":"crossref","unstructured":"Ladosz P, Weng L, Kim M, Oh H (2022) Exploration in deep reinforcement learning: A survey. Inform Fusion 85:1\u201322","key":"5464_CR10","DOI":"10.1016\/j.inffus.2022.03.003"},{"issue":"11","key":"5464_CR11","doi-asserted-by":"publisher","first-page":"3409","DOI":"10.1109\/TNNLS.2019.2891792","volume":"30","author":"N Dilokthanakul","year":"2019","unstructured":"Dilokthanakul N, Kaplanis C, Pawlowski N, Shanahan M (2019) Feature control as intrinsic motivation for hierarchical reinforcement learning. IEEE Trans Neural Netw Learn Syst 30(11):3409\u20133418. https:\/\/doi.org\/10.1109\/TNNLS.2019.2891792","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"issue":"2","key":"5464_CR12","doi-asserted-by":"publisher","first-page":"765","DOI":"10.1109\/TCYB.2022.3157892","volume":"53","author":"H Zhao","year":"2022","unstructured":"Zhao H, Wu J, Li Z, Chen W, Zheng Z (2022) Double sparse deep reinforcement learning via multilayer sparse coding and nonconvex regularized pruning. IEEE Trans Cybern 53(2):765\u2013778","journal-title":"IEEE Trans Cybern"},{"unstructured":"Kipf T, Li Y, Dai H, Zambaldi VF, Sanchez-Gonzalez A, Grefenstette E, Kohli P, Battaglia PW (2019) Compile: Compositional imitation learning and execution. In: Proceedings of the 36th international conference on machine learning, vol 97 pp 3418\u20133428. http:\/\/proceedings.mlr.press\/v97\/kipf19a.html","key":"5464_CR13"},{"key":"5464_CR14","doi-asserted-by":"publisher","first-page":"297","DOI":"10.1146\/ANNUREV-CONTROL-100819-063206","volume":"3","author":"H Ravichandar","year":"2020","unstructured":"Ravichandar H, Polydoros AS, Chernova S (2020) Billard, A. Annual review of control, robotics, and autonomous systems 3:297\u2013330. https:\/\/doi.org\/10.1146\/ANNUREV-CONTROL-100819-063206","journal-title":"Annual review of control, robotics, and autonomous systems"},{"issue":"9","key":"5464_CR15","doi-asserted-by":"publisher","first-page":"14128","DOI":"10.1109\/TITS.2022.3144867","volume":"23","author":"L Le Mero","year":"2022","unstructured":"Le Mero L, Yi D, Dianati M, Mouzakitis A (2022) A survey on imitation learning techniques for end-to-end autonomous vehicles. IEEE Trans Intell Transp Syst 23(9):14128\u201314147","journal-title":"IEEE Trans Intell Transp Syst"},{"unstructured":"Wilcox A, Balakrishna A, Dedieu J, Benslimane W, Brown D, Goldberg K (2022) Monte carlo augmented actor-critic for sparse reward deep reinforcement learning from suboptimal demonstrations. In: Advances in neural information processing systems, vol 35, pp 2254\u20132267","key":"5464_CR16"},{"issue":"7","key":"5464_CR17","doi-asserted-by":"publisher","first-page":"4434","DOI":"10.1007\/S10489-020-02034-2","volume":"51","author":"T Aotani","year":"2021","unstructured":"Aotani T, Kobayashi T, Sugimoto K (2021) Bottom-up multi-agent reinforcement learning by reward shaping for cooperative-competitive tasks. Appl Intell 51(7):4434\u20134452. https:\/\/doi.org\/10.1007\/S10489-020-02034-2","journal-title":"Appl Intell"},{"unstructured":"Hu Y, Wang W, Jia H, Wang Y, Chen Y, Hao J, Wu F, Fan C (2020) Learning to utilize shaping rewards: A new approach of reward shaping. In: Advances in neural information processing systems, vol 33, pp 15931\u201315941","key":"5464_CR18"},{"unstructured":"Sun H, Han L, Yang R, Ma X, Guo J, Zhou B (2022) Exploit reward shifting in value-based deep-rl: Optimistic curiosity-based exploration and conservative exploitation via linear reward shaping. In: Advances in neural information processing systems, vol 35, pp 37719\u201337734","key":"5464_CR19"},{"key":"5464_CR20","doi-asserted-by":"publisher","first-page":"63","DOI":"10.1016\/j.ins.2022.06.050","volume":"608","author":"H Sami","year":"2022","unstructured":"Sami H, Bentahar J, Mourad A, Otrok H, Damiani E (2022) Graph convolutional recurrent networks for reward shaping in reinforcement learning. Inf Sci 608:63\u201380","journal-title":"Inf Sci"},{"key":"5464_CR21","doi-asserted-by":"publisher","DOI":"10.1016\/j.energy.2022.123924","volume":"251","author":"J Li","year":"2022","unstructured":"Li J, Wu X, Xu M, Liu Y (2022) Deep reinforcement learning and reward shaping based eco-driving control for automated hevs among signalized intersections. Energy 251:123924","journal-title":"Energy"},{"unstructured":"Zheng B, Verma S, Zhou J, Tsang IW, Chen F (2022) Imitation learning: Progress, taxonomies and challenges. IEEE Trans on Neural Netw and Learn Syst","key":"5464_CR22"},{"doi-asserted-by":"crossref","unstructured":"Zhu Z, Lin K, Dai B, Zhou J (2022) Self-adaptive imitation learning: Learning tasks with delayed rewards from sub-optimal demonstrations. In: Proceedings of the AAAI conference on artificial intelligence, vol 36, pp 9269\u20139277","key":"5464_CR23","DOI":"10.1609\/aaai.v36i8.20914"},{"issue":"10","key":"5464_CR24","doi-asserted-by":"publisher","first-page":"6968","DOI":"10.1109\/TPAMI.2021.3096966","volume":"44","author":"T Xu","year":"2022","unstructured":"Xu T, Li Z, Yu Y (2022) Error bounds of imitating policies and environments for reinforcement learning. IEEE Trans Pattern Anal Mach Intell 44(10):6968\u20136980. https:\/\/doi.org\/10.1109\/TPAMI.2021.3096966","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"unstructured":"Rolnick D, Ahuja A, Schwarz J, Lillicrap TP, Wayne G (2019) Experience replay for continual learning. In: Advances in neural information processing systems, pp 348\u2013358. https:\/\/proceedings.neurips.cc\/paper\/2019\/hash\/fa7cdfad1a5aaf8370ebeda47a1ff1c3-Abstract.html","key":"5464_CR25"},{"doi-asserted-by":"crossref","unstructured":"Wu J, Huang Z, Huang W, Lv C (2022) Prioritized experience-based reinforcement learning with human guidance for autonomous driving. IEEE Trans Neural Netw Learn Syst 35(1):855\u2013869","key":"5464_CR26","DOI":"10.1109\/TNNLS.2022.3177685"},{"unstructured":"Packer C, Abbeel P, Gonzalez JE (2021) Hindsight task relabelling: Experience replay for sparse reward meta-rl. In: Advances in neural information processing systems, pp 2466\u20132477. https:\/\/proceedings.neurips.cc\/paper\/2021\/hash\/1454ca2270599546dfcd2a3700e4d2f1-Abstract.html","key":"5464_CR27"},{"unstructured":"Weng W, Gupta H, He N, Ying L, Srikant R (2020) The mean-squared error of double q-learning. In: Advances in neural information processing systems, vol 33, pp 6815\u20136826. https:\/\/proceedings.neurips.cc\/paper\/2020\/hash\/4bfbd52f4e8466dc12aaf30b7e057b66-Abstract.html","key":"5464_CR28"},{"unstructured":"Yang Y, Hao J, Chen G, Tang H, Chen Y, Hu Y, Fan C, Wei Z (2020) Q-value path decomposition for deep multiagent reinforcement learning. In: Proceedings of the 37th international conference on machine learning, vol 119, pp 10706\u201310715. http:\/\/proceedings.mlr.press\/v119\/yang20d.html","key":"5464_CR29"},{"unstructured":"Fujimoto S, Hoof H, Meger D (2018) Addressing function approximation error in actor-critic methods. In: Proceedings of the 35th international conference on machine learning, vol 80, pp 1582\u20131591. http:\/\/proceedings.mlr.press\/v80\/fujimoto18a.html","key":"5464_CR30"},{"doi-asserted-by":"publisher","unstructured":"Dabney W, Rowland M, Bellemare MG, Munos R (2018) Distributional reinforcement learning with quantile regression. In: Proceedings of the thirty-second AAAI conference on artificial intelligence, pp 2892\u20132901. https:\/\/doi.org\/10.1609\/AAAI.V32I1.11791","key":"5464_CR31","DOI":"10.1609\/AAAI.V32I1.11791"},{"unstructured":"Kapturowski S, Ostrovski G, Quan J, Munos R, Dabney W (2019) Recurrent experience replay in distributed reinforcement learning. In: International conference on learning representations . https:\/\/openreview.net\/forum?id=r1lyTjAqYX","key":"5464_CR32"},{"unstructured":"Sutton RS, Barto AG (2018) Reinforcement Learning: An Introduction, Cambridge","key":"5464_CR33"},{"unstructured":"Haarnoja T, Zhou A, Abbeel P, Levine S (2018) Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor. In: Proceedings of the 35th international conference on machine learning, vol 80, pp 1856\u20131865. http:\/\/proceedings.mlr.press\/v80\/haarnoja18b.html","key":"5464_CR34"},{"unstructured":"Lee SY, Choi S, Chung S (2019) Sample-efficient deep reinforcement learning via episodic backward update. In: Advances in neural information processing systems, pp 2110\u20132119. https:\/\/proceedings.neurips.cc\/paper\/2019\/hash\/e6d8545daa42d5ced125a4bf747b3688-Abstract.html","key":"5464_CR35"}],"container-title":["Applied Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-024-05464-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10489-024-05464-4\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-024-05464-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,6,15]],"date-time":"2024-06-15T12:08:33Z","timestamp":1718453313000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10489-024-05464-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4]]},"references-count":35,"journal-issue":{"issue":"8","published-print":{"date-parts":[[2024,4]]}},"alternative-id":["5464"],"URL":"https:\/\/doi.org\/10.1007\/s10489-024-05464-4","relation":{},"ISSN":["0924-669X","1573-7497"],"issn-type":[{"type":"print","value":"0924-669X"},{"type":"electronic","value":"1573-7497"}],"subject":[],"published":{"date-parts":[[2024,4]]},"assertion":[{"value":"11 April 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"3 May 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors confirm that this research project has undergone rigorous ethical review and has received approval.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics approval"}},{"value":"The authors declare no conflicts of interest that could influence the research design, data collection, or interpretation of results.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}