{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T01:25:59Z","timestamp":1775265959959,"version":"3.50.1"},"reference-count":104,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"Institute of Information & communications Technology Planning & Evaluation"},{"name":"Korea government","award":["2022-0-00124"],"award-info":[{"award-number":["2022-0-00124"]}]},{"name":"Development of Artificial Intelligence Technology for Self-Improving Competency-Aware Learning Capabilities"},{"DOI":"10.13039\/501100003725","name":"National Research Foundation of Korea","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100003725","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Korea government","award":["NRF-2022R1A2C2005529"],"award-info":[{"award-number":["NRF-2022R1A2C2005529"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE\/ACM Trans. Audio Speech Lang. Process."],"published-print":{"date-parts":[[2024]]},"DOI":"10.1109\/taslp.2024.3444470","type":"journal-article","created":{"date-parts":[[2024,8,15]],"date-time":"2024-08-15T17:45:43Z","timestamp":1723743943000},"page":"3934-3946","source":"Crossref","is-referenced-by-count":12,"title":["Textless Unit-to-Unit Training for Many-to-Many Multilingual Speech-to-Speech Translation"],"prefix":"10.1109","volume":"32","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6514-0018","authenticated-orcid":false,"given":"Minsu","family":"Kim","sequence":"first","affiliation":[{"name":"Integrated Vision and Language Laboratory, School of Electrical Engineering, Korea Advanced Institute of Science and Technology (KAIST), Daejeon, South Korea"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-6817-604X","authenticated-orcid":false,"given":"Jeongsoo","family":"Choi","sequence":"additional","affiliation":[{"name":"Integrated Vision and Language Laboratory, School of Electrical Engineering, Korea Advanced Institute of Science and Technology (KAIST), Daejeon, South Korea"}]},{"given":"Dahun","family":"Kim","sequence":"additional","affiliation":[{"name":"Google DeepMind, Mountain View, CA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5306-6853","authenticated-orcid":false,"given":"Yong Man","family":"Ro","sequence":"additional","affiliation":[{"name":"Integrated Vision and Language Laboratory, School of Electrical Engineering, Korea Advanced Institute of Science and Technology (KAIST), Daejeon, South Korea"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Deep speech: Scaling up end-to-end speech recognition","author":"Hannun","year":"2014"},{"key":"ref2","first-page":"173","article-title":"Deep speech 2: End-to-end speech recognition in english and mandarin","volume-title":"Proc. Int. Conf. Mach. Learn","author":"Amodei","year":"2016"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2017.7953075"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/JSTSP.2017.2763455"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.21437\/interspeech.2018-1456"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2021.3115626"},{"key":"ref7","article-title":"Learning audio-visual speech representation by masked multimodal cluster prediction","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Shi","year":"2022"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i1.20003"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01801"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1146\/annurev.neuro.26.041002.131047"},{"key":"ref11","article-title":"Googles neural machine translation system: Bridging the gap between human and machine translation","author":"Wu","year":"2016"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1008"},{"key":"ref13","first-page":"1877","article-title":"Language models are few-shot learners","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Brown","year":"2020"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00343"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2017-1452"},{"key":"ref16","first-page":"4485","article-title":"Transfer learning from speaker verification to multispeaker text-to-speech synthesis","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"31","author":"Jia","year":"2018"},{"key":"ref17","article-title":"Flowtron: An autoregressive flow-based generative network for text-to-speech synthesis","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Valle","year":"2021"},{"key":"ref18","article-title":"Adaspeech: Adaptive text to speech for custom voice","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Chen","year":"2021"},{"key":"ref19","first-page":"2709","article-title":"YourTTS: Towards zero-shot multi-speaker tts and zero-shot voice conversion for everyone","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Casanova","year":"2022"},{"key":"ref20","article-title":"Neural codec language models are zero-shot text to speech synthesizers","author":"Wang","year":"2023"},{"key":"ref21","article-title":"Lamda: Language models for dialog applications","author":"Thoppilan","year":"2022"},{"key":"ref22","article-title":"Llama: Open and efficient foundation language models","author":"Touvron","year":"2023"},{"key":"ref23","article-title":"Openai: Introducing chatGPT","year":"2022"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.21437\/interspeech.2019-2668"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2020-2679"},{"key":"ref26","article-title":"Speak foreign languages with your own voice: Cross-lingual neural codec language modeling","author":"Zhang","year":"2023"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2023\/575"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1162"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1007"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/d18-2012"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.21105\/joss.03958"},{"key":"ref32","first-page":"1336","article-title":"On generative spoken language modeling from raw audio","volume":"9","author":"Lakhotia","year":"2021","journal-title":"Trans. Assoc. Comput. Linguistics"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2021-475"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2023-194"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2021.3122291"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2023-2051"},{"key":"ref37","first-page":"12449","article-title":"wav2vec 2.0: A framework for self-supervised learning of speech representations","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Baevski","year":"2020"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2022-143"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2023.3288409"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1016\/S0167-6393(00)00094-7"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2014.6855086"},{"key":"ref42","article-title":"Synchronous bidirectional learning for multilingual lip reading","volume-title":"Proc. Brit. Mach. Vis. Conf.","author":"Luo","year":"2020"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.naacl-main.63"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/JSTSP.2022.3188113"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.393"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.367"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2021.3126925"},{"key":"ref48","article-title":"Prompt tuning of deep neural networks for speaker-adaptive visual speech recognition","author":"Kim","year":"2023"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP49357.2023.10094693"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP48485.2024.10447929"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00545"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.769"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2016.2643280"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2014.2304637"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2014.2339736"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2020.2988423"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2018.2789721"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2022.3221040"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2022.3167258"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2023.3308374"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-1951"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2020.2986886"},{"key":"ref63","first-page":"10120","article-title":"Translatotron 2: High-quality direct speech-to-speech translation with voice preservation","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Jia","year":"2022"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/SLT48900.2021.9383496"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref66","article-title":"Transpeech: Speech-to-speech translation with bilateral perturbation","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Huang","year":"2023"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.872"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.235"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2022-10938"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/ASRU51503.2021.9688253"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2022-11032"},{"key":"ref72","article-title":"Textless low-resource speech-to-speech translation with unit language models","author":"Diwan","year":"2023"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1009"},{"key":"ref74","article-title":"Seamlessm4t-massively multilingual & multimodal machine translation","author":"Barrault","year":"2023"},{"key":"ref75","first-page":"3171","article-title":"Fastspeech: Fast, robust and controllable text to speech","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"32","author":"Ren","year":"2019"},{"key":"ref76","first-page":"195","article-title":"Deep voice: Real-time neural text-to-speech","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Arik","year":"2017"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9054535"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2020.3027619"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3403331"},{"key":"ref80","article-title":"Multilingual byte2speech models for scalable low-resource speech synthesis","author":"He","year":"2021"},{"key":"ref81","first-page":"741","article-title":"Low-resource multilingual and zero-shot multispeaker TTS","volume-title":"Proc. 2nd Conf. Asia-Pacific Chap. Assoc. Comput. Linguistics 12th Int. Joint Conf. Natural Lang. Process.","author":"Lux","year":"2022"},{"key":"ref82","article-title":"High fidelity neural audio compression","volume-title":"Proc. Trans. Mach. Learn. Res.","author":"Dfossez","year":"2023"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP49357.2023.10097097"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.108"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP49357.2023.10096797"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.80"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.703"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1810.04805"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-4009"},{"key":"ref90","first-page":"17022","article-title":"Hifi-GAN: Generative adversarial networks for efficient and high fidelity speech synthesis","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Kong","year":"2020"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2021-11"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2023-2279"},{"key":"ref93","first-page":"5530","article-title":"Conditional variational autoencoder with adversarial learning for end-to-end text-to-speech","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Kim","year":"2021"},{"key":"ref94","article-title":"The lj speech dataset","author":"Ito","year":"2017"},{"key":"ref95","first-page":"186","article-title":"A call for clarity in reporting BLEU scores","volume-title":"Proc. 3rd Conf. Mach. Translation, Res. Papers","author":"Post","year":"2018"},{"key":"ref96","first-page":"46","volume-title":"Proc. 7th Conf. Mach. Transl.","author":"Freitag","year":"2022"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.899"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-1500"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9054626"},{"key":"ref100","first-page":"6691","article-title":"Cvss corpus and massively multilingual speech-to-speech translation","volume-title":"Proc. 13th Lang. Resour. Eval. Conf.","author":"Jia","year":"2022"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2021-2027"},{"key":"ref102","article-title":"Mslam: Massively multilingual joint pre-training for speech and text","author":"Bapna","year":"2022"},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2021-1757"},{"key":"ref104","article-title":"Lrs3-ted: A large-scale dataset for visual speech recognition","author":"Afouras","year":"2018"}],"container-title":["IEEE\/ACM Transactions on Audio, Speech, and Language Processing"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/6570655\/10304349\/10637752.pdf?arnumber=10637752","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T04:52:55Z","timestamp":1726030375000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10637752\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"references-count":104,"URL":"https:\/\/doi.org\/10.1109\/taslp.2024.3444470","relation":{},"ISSN":["2329-9290","2329-9304"],"issn-type":[{"value":"2329-9290","type":"print"},{"value":"2329-9304","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024]]}}}