{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,26]],"date-time":"2025-03-26T10:46:43Z","timestamp":1742986003567,"version":"3.40.3"},"publisher-location":"Cham","reference-count":45,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031723407"},{"type":"electronic","value":"9783031723414"}],"license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024]]},"DOI":"10.1007\/978-3-031-72341-4_27","type":"book-chapter","created":{"date-parts":[[2024,9,16]],"date-time":"2024-09-16T13:02:55Z","timestamp":1726491775000},"page":"397-411","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Enhancing Visual Generalization in\u00a0Reinforcement Learning with\u00a0Cycling Augmentation"],"prefix":"10.1007","author":[{"given":"Shengjie","family":"Sun","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jiafei","family":"Lyu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Lu","family":"Li","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jiazhe","family":"Guo","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mengbei","family":"Yan","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Runze","family":"Liu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xiu","family":"Li","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,9,17]]},"reference":[{"key":"27_CR1","unstructured":"Agarwal, R., Machado, M.C., Castro, P.S., Bellemare, M.G.: Contrastive behavioral similarity embeddings for generalization in reinforcement learning (2021). arXiv preprint arXiv:2101.05265"},{"key":"27_CR2","unstructured":"Beattie, C., et\u00a0al.: Deepmind lab (2016). arXiv preprint arXiv:1612.03801"},{"key":"27_CR3","doi-asserted-by":"crossref","unstructured":"Bellman, R.: A markovian decision process. J. Math. Mech. pp. 679\u2013684 (1957)","DOI":"10.1512\/iumj.1957.6.56038"},{"key":"27_CR4","first-page":"30693","volume":"35","author":"D Bertoin","year":"2022","unstructured":"Bertoin, D., Zouitine, A., Zouitine, M., Rachelson, E.: Look where you look! saliency-guided Q-networks for generalization in visual reinforcement learning. Adv. Neural. Inf. Process. Syst. 35, 30693\u201330706 (2022)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"27_CR5","doi-asserted-by":"crossref","unstructured":"Chebotar, Y., et al.: Closing the sim-to-real loop: adapting simulation randomization with real world experience. In: 2019 International Conference on Robotics and Automation (ICRA), pp. 8973\u20138979. IEEE (2019)","DOI":"10.1109\/ICRA.2019.8793789"},{"key":"27_CR6","doi-asserted-by":"crossref","unstructured":"Cubuk, E.D., Zoph, B., Mane, D., Vasudevan, V., Le, Q.V.: Autoaugment: learning augmentation strategies from data. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 113\u2013123 (2019)","DOI":"10.1109\/CVPR.2019.00020"},{"key":"27_CR7","doi-asserted-by":"crossref","unstructured":"Cubuk, E.D., Zoph, B., Shlens, J., Le, Q.V.: Randaugment: practical automated data augmentation with a reduced search space. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition Workshops, pp. 702\u2013703 (2020)","DOI":"10.1109\/CVPRW50498.2020.00359"},{"key":"27_CR8","unstructured":"Cubuk, E.D., Dyer, E.S., Lopes, R.G., Smullin, S.: Tradeoffs in data augmentation: an empirical study. In: Proceedings of the International Conference on Learning Representations (2021)"},{"key":"27_CR9","unstructured":"Du, S., Krishnamurthy, A., Jiang, N., Agarwal, A., Dudik, M., Langford, J.: Provably efficient rl with rich observations via latent state decoding. In: International Conference on Machine Learning, pp. 1665\u20131674. PMLR (2019)"},{"key":"27_CR10","unstructured":"Fan, L., et al.: Secant: self-expert cloning for zero-shot generalization of visual policies (2021). arXiv preprint arXiv:2106.09678"},{"key":"27_CR11","unstructured":"Hansen, N., et al.: Self-supervised policy adaptation during deployment (2020). arXiv preprint arXiv:2007.04309"},{"key":"27_CR12","first-page":"3680","volume":"34","author":"N Hansen","year":"2021","unstructured":"Hansen, N., Su, H., Wang, X.: Stabilizing deep q-learning with convnets and vision transformers under data augmentation. Adv. Neural. Inf. Process. Syst. 34, 3680\u20133693 (2021)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"27_CR13","unstructured":"Hendrycks, D., Mu, N., Cubuk, E.D., Zoph, B., Gilmer, J., Lakshminarayanan, B.: Augmix: A simple data processing method to improve robustness and uncertainty (2019). arXiv preprint arXiv:1912.02781"},{"key":"27_CR14","doi-asserted-by":"crossref","unstructured":"Hendrycks, D., Zou, A., Mazeika, M., Tang, L., Li, B., Song, D., Steinhardt, J.: Pixmix: dreamlike pictures comprehensively improve safety measures. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 16783\u201316792 (2022)","DOI":"10.1109\/CVPR52688.2022.01628"},{"issue":"1\u20132","key":"27_CR15","doi-asserted-by":"publisher","first-page":"99","DOI":"10.1016\/S0004-3702(98)00023-X","volume":"101","author":"LP Kaelbling","year":"1998","unstructured":"Kaelbling, L.P., Littman, M.L., Cassandra, A.R.: Planning and acting in partially observable stochastic domains. Artif. Intell. 101(1\u20132), 99\u2013134 (1998)","journal-title":"Artif. Intell."},{"key":"27_CR16","unstructured":"Kingma, D.P., Ba, J.: Adam: A method for stochastic optimization (2014). arXiv preprint arXiv:1412.6980"},{"key":"27_CR17","first-page":"19884","volume":"33","author":"M Laskin","year":"2020","unstructured":"Laskin, M., Lee, K., Stooke, A., Pinto, L., Abbeel, P., Srinivas, A.: Reinforcement learning with augmented data. Adv. Neural. Inf. Process. Syst. 33, 19884\u201319895 (2020)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"27_CR18","unstructured":"Lee, K., Lee, K., Shin, J., Lee, H.: Network randomization: A simple technique for generalization in deep reinforcement learning (2019). arXiv preprint arXiv:1910.05396"},{"key":"27_CR19","unstructured":"Li, L., Spratling, M.: Data augmentation alone can improve adversarial training (2023). arXiv preprint arXiv:2301.09879"},{"key":"27_CR20","unstructured":"Li, L., et al.: Normalization enhances generalization in visual reinforcement learning (2023). arXiv preprint arXiv:2306.00656"},{"key":"27_CR21","unstructured":"LingChen, T.C., Khonsari, A., Lashkari, A., Nazari, M.R., Sambee, J.S., Nascimento, M.A.: Uniformaugment: A search-free probabilistic data augmentation approach (2020). arXiv preprint arXiv:2003.14348"},{"key":"27_CR22","unstructured":"Lyu, J., Wan, L., Li, X., Lu, Z.: Towards understanding how to reduce generalization gap in visual reinforcement learning. In: Proceedings of the 23rd International Conference on Autonomous Agents and Multiagent Systems. pp. 2369\u20132371 (2024)"},{"key":"27_CR23","doi-asserted-by":"crossref","unstructured":"Lyu, J., Wan, L., Li, X., Lu, Z.: Understanding what affects generalization gap in visual reinforcement learning: Theory and empirical evidence (2024). arXiv preprint arXiv:2402.02701","DOI":"10.1613\/jair.1.16422"},{"key":"27_CR24","unstructured":"Ma, G., et al.: Learning better with less: effective augmentation for sample-efficient visual reinforcement learning. Adv. Neural Inf. Proc. Syst. 36 (2024)"},{"key":"27_CR25","doi-asserted-by":"crossref","unstructured":"Mitrano, P., Berenson, D.: Data augmentation for manipulation (2022). arXiv preprint arXiv:2205.02886","DOI":"10.15607\/RSS.2022.XVIII.031"},{"key":"27_CR26","doi-asserted-by":"crossref","unstructured":"Mnih, V., et\u00a0al.: Human-level control through deep reinforcement learning. Nature 518(7540), 529\u2013533 (2015)","DOI":"10.1038\/nature14236"},{"key":"27_CR27","doi-asserted-by":"crossref","unstructured":"M\u00fcller, S.G., Hutter, F.: Trivialaugment: Tuning-free yet state-of-the-art data augmentation. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 774\u2013782 (2021)","DOI":"10.1109\/ICCV48922.2021.00081"},{"key":"27_CR28","unstructured":"Nair, A.V., Pong, V., Dalal, M., Bahl, S., Lin, S., Levine, S.: Visual reinforcement learning with imagined goals. Adv. Neural Inf. Proc. Syst. 31 (2018)"},{"key":"27_CR29","unstructured":"Packer, C., Gao, K., Kos, J., Kr\u00e4henb\u00fchl, P., Koltun, V., Song, D.: Assessing generalization in deep reinforcement learning (2018). arXiv preprint arXiv:1810.12282"},{"key":"27_CR30","doi-asserted-by":"crossref","unstructured":"Peng, X.B., Andrychowicz, M., Zaremba, W., Abbeel, P.: Sim-to-real transfer of robotic control with dynamics randomization. In: 2018 IEEE International Conference on Robotics and Automation (ICRA), pp. 3803\u20133810. IEEE (2018)","DOI":"10.1109\/ICRA.2018.8460528"},{"key":"27_CR31","doi-asserted-by":"crossref","unstructured":"Pinto, L., Andrychowicz, M., Welinder, P., Zaremba, W., Abbeel, P.: Asymmetric actor critic for image-based robot learning (2017). arXiv preprint arXiv:1710.06542","DOI":"10.15607\/RSS.2018.XIV.008"},{"key":"27_CR32","doi-asserted-by":"crossref","unstructured":"Ramos, F., Possas, R.C., Fox, D.: Bayessim: adaptive domain randomization via probabilistic inference for robotics simulators (2019). arXiv preprint arXiv:1906.01728","DOI":"10.15607\/RSS.2019.XV.029"},{"key":"27_CR33","unstructured":"Ratner, A.J., Ehrenberg, H., Hussain, Z., Dunnmon, J., R\u00e9, C.: Learning to compose domain-specific transformations for data augmentation. Adv. Neural Inf. Proc. Syst. 30 (2017)"},{"key":"27_CR34","doi-asserted-by":"crossref","unstructured":"Suzuki, T.: Teachaugment: data augmentation optimization using teacher knowledge. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 10904\u201310914 (2022)","DOI":"10.1109\/CVPR52688.2022.01063"},{"key":"27_CR35","doi-asserted-by":"crossref","unstructured":"Tobin, J., Fong, R., Ray, A., Schneider, J., Zaremba, W., Abbeel, P.: Domain randomization for transferring deep neural networks from simulation to the real world. In: 2017 IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS), pp. 23\u201330. IEEE (2017)","DOI":"10.1109\/IROS.2017.8202133"},{"key":"27_CR36","first-page":"13456","volume":"35","author":"H Wang","year":"2022","unstructured":"Wang, H., Lohit, S., Jones, M.N., Fu, Y.: What makes a \u201cgood\u2019\u2019 data augmentation in knowledge distillation-a statistical perspective. Adv. Neural. Inf. Process. Syst. 35, 13456\u201313469 (2022)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"27_CR37","first-page":"7968","volume":"33","author":"K Wang","year":"2020","unstructured":"Wang, K., Kang, B., Shao, J., Feng, J.: Improving generalization in reinforcement learning with mixture regularization. Adv. Neural. Inf. Process. Syst. 33, 7968\u20137978 (2020)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"27_CR38","unstructured":"Yarats, D., Fergus, R., Lazaric, A., Pinto, L.: Mastering visual continuous control: Improved data-augmented reinforcement learning (2021). arXiv preprint arXiv:2107.09645"},{"key":"27_CR39","unstructured":"Yarats, D., Kostrikov, I., Fergus, R.: Image augmentation is all you need: regularizing deep reinforcement learning from pixels. In: International Conference on Learning Representations (2020)"},{"key":"27_CR40","doi-asserted-by":"crossref","unstructured":"Yuan, Z., et al.: Don\u2019t touch what matters: Task-aware lipschitz data augmentation for visual reinforcement learning (2022). arXiv preprint arXiv:2202.09982","DOI":"10.24963\/ijcai.2022\/514"},{"key":"27_CR41","first-page":"13022","volume":"35","author":"Z Yuan","year":"2022","unstructured":"Yuan, Z., et al.: Pre-trained image encoder for generalizable visual reinforcement learning. Adv. Neural. Inf. Process. Syst. 35, 13022\u201313037 (2022)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"27_CR42","unstructured":"Yuan, Z., Yang, S., Hua, P., Chang, C., Hu, K., Xu, H.: Rl-vigen: a reinforcement learning benchmark for visual generalization. Adv. Neural Inf. Proc. Syst. 36 (2024)"},{"key":"27_CR43","doi-asserted-by":"crossref","unstructured":"Yun, S., Han, D., Oh, S.J., Chun, S., Choe, J., Yoo, Y.: Cutmix: regularization strategy to train strong classifiers with localizable features. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 6023\u20136032 (2019)","DOI":"10.1109\/ICCV.2019.00612"},{"key":"27_CR44","unstructured":"Zhang, H., Cisse, M., Dauphin, Y.N., Lopez-Paz, D.: mixup: Beyond empirical risk minimization (2017). arXiv preprint arXiv:1710.09412"},{"key":"27_CR45","doi-asserted-by":"crossref","unstructured":"Zhu, Y., Mottaghi, R., Kolve, E., Lim, J.J., Gupta, A., Fei-Fei, L., Farhadi, A.: Target-driven visual navigation in indoor scenes using deep reinforcement learning. In: 2017 IEEE International Conference on Robotics and Automation (ICRA), pp. 3357\u20133364. IEEE (2017)","DOI":"10.1109\/ICRA.2017.7989381"}],"container-title":["Lecture Notes in Computer Science","Artificial Neural Networks and Machine Learning \u2013 ICANN 2024"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-72341-4_27","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,16]],"date-time":"2024-09-16T13:13:58Z","timestamp":1726492438000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-72341-4_27"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"ISBN":["9783031723407","9783031723414"],"references-count":45,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-72341-4_27","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2024]]},"assertion":[{"value":"17 September 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors have no competing interests to declare that are relevant to the content of this article.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"ICANN","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Artificial Neural Networks","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Lugano","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Switzerland","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"17 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"20 September 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"33","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"icann2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}