{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T07:29:28Z","timestamp":1740122968017,"version":"3.37.3"},"reference-count":37,"publisher":"Springer Science and Business Media LLC","issue":"35","license":[{"start":{"date-parts":[[2024,3,11]],"date-time":"2024-03-11T00:00:00Z","timestamp":1710115200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,3,11]],"date-time":"2024-03-11T00:00:00Z","timestamp":1710115200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Multimed Tools Appl"],"DOI":"10.1007\/s11042-024-18695-x","type":"journal-article","created":{"date-parts":[[2024,3,11]],"date-time":"2024-03-11T07:01:31Z","timestamp":1710140491000},"page":"82019-82033","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Audio\u2013video syncing with lip movements using generative deep neural networks"],"prefix":"10.1007","volume":"83","author":[{"given":"Amal","family":"Mathew","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Aaryl","family":"Saldanha","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2358-3443","authenticated-orcid":false,"given":"C. Narendra","family":"Babu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,3,11]]},"reference":[{"key":"18695_CR1","unstructured":"Van den Oord A, Dieleman S, Zen H, Simonyan K, Vinyals O, Graves A, Kalchbrenner N, Senior A, Kavukcuoglu K (2016) WaveNet: A generative model for raw audio. arXiv preprint arXiv:1609.03499.\u00a0 Accessed 19 Sep 2016"},{"key":"18695_CR2","doi-asserted-by":"publisher","first-page":"781","DOI":"10.1109\/SIU.2011.5929767","volume-title":"2011 IEEE 19th Signal Processing and Communications Applications Conference (SIU)","author":"MC Orhan","year":"2011","unstructured":"Orhan MC, Demiro\u011flu C (2011) HMM-based text to speech system with speaker interpolation. 2011 IEEE 19th Signal Processing and Communications Applications Conference (SIU). IEEE, pp 781\u2013784"},{"key":"18695_CR3","first-page":"195","volume-title":"International Conference on Machine Learning","author":"S\u00d6 Ar\u0131k","year":"2017","unstructured":"Ar\u0131k S\u00d6, Chrzanowski M, Coates A, Diamos G, Gibiansky A, Kang Y, Li X et al (2017) Deep voice: Real-time neural text-to-speech. International Conference on Machine Learning. PMLR, pp 195\u2013204"},{"key":"18695_CR4","unstructured":"Sotelo J, Mehri S, Kumar K, Santos JF, Kastner K, Courville A, Bengio Y (2017) Char2wav: End-to-end speech synthesis. arXiv preprint arXiv:1702.07825.\u00a0Accessed 7 Mar 2017"},{"key":"18695_CR5","unstructured":"Mehri S, Kumar K, Gulrajani I, Kumar R, Jain S, Sotelo J, Courville A, Bengio Y (2016) SampleRNN: An unconditional end-to-end neural audio generation model. arXiv preprint arXiv:1612.07837. Accessed 11 Feb 2017"},{"key":"18695_CR6","doi-asserted-by":"crossref","unstructured":"Wang Y, Skerry-Ryan RJ, Stanton D, Wu Y, Weiss RJ, Jaitly N, Yang Z, et al (2017) Tacotron: Towards end-to-end speech synthesis. arXiv preprint arXiv:1703.10135. Accessed 6 Apr 2017","DOI":"10.21437\/Interspeech.2017-1452"},{"issue":"2","key":"18695_CR7","doi-asserted-by":"publisher","first-page":"236","DOI":"10.1109\/TASSP.1984.1164317","volume":"32","author":"D Griffin","year":"1984","unstructured":"Griffin D, Lim J (1984) Signal estimation from modified short-time Fourier transform. IEEE Trans Acoust Speech Signal Process 32(2):236\u2013243","journal-title":"IEEE Trans Acoust Speech Signal Process"},{"key":"18695_CR8","unstructured":"Ping W, Peng K, Chen J (2018) Clarinet: Parallel wave generation in end-to-end text-to-speech. arXiv preprint arXiv:1807.07281.\u00a0Accessed 27 Apr 2017"},{"key":"18695_CR9","doi-asserted-by":"publisher","first-page":"3617","DOI":"10.1109\/ICASSP.2019.8683143","volume-title":"ICASSP 2019\u20132019 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","author":"R Prenger","year":"2019","unstructured":"Prenger R, Valle R, Catanzaro B (2019) Waveglow: A flow-based generative network for speech synthesis. ICASSP 2019\u20132019 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP). IEEE, pp 3617\u20133621"},{"key":"18695_CR10","unstructured":"Kingma DP, Dhariwal P (2018) Glow: Generative flow with invertible 1x1 convolutions. ArXiv, abs\/1807.03039."},{"issue":"4","key":"18695_CR11","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3072959.3073640","volume":"36","author":"S Suwajanakorn","year":"2017","unstructured":"Suwajanakorn S, Seitz SM, Kemelmacher-Shlizerman I (2017) Synthesizing obama: learning lip sync from audio. ACM Trans Graph (TOG) 36(4):1\u201313","journal-title":"ACM Trans Graph (TOG)"},{"key":"18695_CR12","unstructured":"Kumar R, Sotelo J, Kumar K, de Br\u00e9bisson A, Bengio Y (2017) Obamanet: Photo-realistic lip-sync from text. arXiv preprint arXiv:1801.01442.\u00a0Accessed 6 Dec 2017"},{"key":"18695_CR13","doi-asserted-by":"crossref","unstructured":"Isola P, Zhu JY, Zhou T, Efros AA (2017) Image-to-image translation with conditional adversarial networks. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR), Honolulu, HI, USA, 2017, pp 5967\u20135976","DOI":"10.1109\/CVPR.2017.632"},{"key":"18695_CR14","unstructured":"Chung JS, Jamaludin A, Zisserman A (2017) You said that?. arXiv preprint arXiv:1705.02966.\u00a0Accessed\u00a0 18 Jul 2017"},{"key":"18695_CR15","doi-asserted-by":"publisher","unstructured":"KR P, Mukhopadhyay R, Philip J, Jha A, Namboodiri V, Jawahar CV (2019) Towards automatic face-to-face translation. In: Proceedings of the 27th ACM International Conference on Multimedia, pp 1428\u20131436.\u00a0https:\/\/doi.org\/10.1145\/3343031.3351066","DOI":"10.1145\/3343031.3351066"},{"key":"18695_CR16","doi-asserted-by":"publisher","unstructured":"Prajwal KR, Mukhopadhyay R, Namboodiri VP, Jawahar CV (2020) A lip sync expert is all you need for speech to lip generation in the wild. In: Proceedings of the 28th ACM International Conference on Multimedia, pp. 484\u2013492. https:\/\/doi.org\/10.1145\/3394171.3413532","DOI":"10.1145\/3394171.3413532"},{"key":"18695_CR17","first-page":"251","volume-title":"Asian Conference on Computer Vision","author":"JS Chung","year":"2016","unstructured":"Chung JS, Zisserman A (2016) Out of time: automated lip sync in the wild. Asian Conference on Computer Vision. Springer, Cham, pp 251\u2013263"},{"key":"18695_CR18","first-page":"405","volume-title":"European Conference on Computer Vision","author":"B Mildenhall","year":"2020","unstructured":"Mildenhall B, Srinivasan PP, Tancik M, Barron JT, Ramamoorthi R, Ng R (2020) Nerf: Representing scenes as neural radiance fields for view synthesis. European Conference on Computer Vision. Springer, Cham, pp 405\u2013421"},{"key":"18695_CR19","doi-asserted-by":"publisher","unstructured":"Guo Y, Chen K, Liang S, Liu YJ, Bao H, Zhang J (2021) Ad-nerf: Audio driven neural radiance fields for talking head synthesis. In: 2021 IEEE\/CVF International Conference on Computer Vision (ICCV), Montreal, QC, Canada, pp 5764\u20135774. https:\/\/doi.org\/10.1109\/ICCV48922.2021.00573","DOI":"10.1109\/ICCV48922.2021.00573"},{"key":"18695_CR20","unstructured":"Yao S, Zhong R, Yan Y, Zhai G, Yang X (2022) DFA-NeRF: Personalized Talking Head Generation via Disentangled Face Attributes Neural Rendering. arXiv preprint arXiv:2201.00791.\u00a0Accessed 3 Jan 2022"},{"key":"18695_CR21","doi-asserted-by":"publisher","first-page":"7140","DOI":"10.1109\/ICASSP.2019.8682275","volume-title":"ICASSP 2019\u20132019 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","author":"A Jha","year":"2019","unstructured":"Jha A, Voleti V, Namboodiri V, Jawahar CV (2019) Cross-language Speech Dependent Lip-synchronization. ICASSP 2019\u20132019 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP). IEEE, pp 7140\u20137144"},{"issue":"1\u20132","key":"18695_CR22","doi-asserted-by":"publisher","first-page":"23","DOI":"10.1016\/S0167-6393(98)00048-X","volume":"26","author":"H Yehia","year":"1998","unstructured":"Yehia H, Rubin P, Vatikiotis-Bateson E (1998) Quantitative association of vocal-tract and facial behavior. Speech Commun 26(1\u20132):23\u201343","journal-title":"Speech Commun"},{"issue":"4","key":"18695_CR23","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3072959.3073658","volume":"36","author":"T Karras","year":"2017","unstructured":"Karras T, Aila T, Laine S, Herva A, Lehtinen J (2017) Audio-driven facial animation by joint end-to-end learning of pose and emotion. ACM Trans Graph (TOG) 36(4):1\u201312","journal-title":"ACM Trans Graph (TOG)"},{"key":"18695_CR24","doi-asserted-by":"publisher","unstructured":"Li Y, Min M, Shen D, Carlson D, Carin L (2018) Video generation from text. AAAI Conf Artif Intell 32(1). https:\/\/doi.org\/10.1609\/aaai.v32i1.12233","DOI":"10.1609\/aaai.v32i1.12233"},{"key":"18695_CR25","first-page":"173","volume-title":"International Conference on Machine Learning","author":"D Amodei","year":"2016","unstructured":"Amodei D, Ananthanarayanan S, Anubhai R, Bai J, Battenberg E, Case C, Casper J et al (2016) Deep speech 2: End-to-end speech recognition in English and Mandarin. International Conference on Machine Learning. PMLR, pp 173\u2013182"},{"key":"18695_CR26","doi-asserted-by":"publisher","unstructured":"Lee CH, Liu Z, Wu L, Luo P (2020) MaskGAN: Towards diverse and interactive facial image manipulation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition , Seattle, WA, USA, pp 5548\u20135557. https:\/\/doi.org\/10.1109\/CVPR42600.2020.00559","DOI":"10.1109\/CVPR42600.2020.00559"},{"key":"18695_CR27","first-page":"694","volume-title":"European Conference on Computer Vision","author":"J Johnson","year":"2016","unstructured":"Johnson J, Alahi A, Fei-Fei L (2016) Perceptual losses for real-time style transfer and super-resolution. European Conference on Computer Vision. Springer, Cham, pp 694\u2013711"},{"issue":"1","key":"18695_CR28","doi-asserted-by":"publisher","first-page":"47","DOI":"10.1109\/TCI.2016.2644865","volume":"3","author":"H Zhao","year":"2016","unstructured":"Zhao H, Gallo O, Frosio I, Kautz J (2016) Loss functions for image restoration with neural networks. IEEE Trans Comput Imaging 3(1):47\u201357","journal-title":"IEEE Trans Comput Imaging"},{"key":"18695_CR29","doi-asserted-by":"publisher","first-page":"1477","DOI":"10.1109\/ICIP.2012.6467150","volume-title":"2012 19th IEEE International Conference on Image Processing","author":"L Zhang","year":"2012","unstructured":"Zhang L, Zhang L, Mou X, Zhang D (2012) A comprehensive evaluation of full reference image quality assessment algorithms. 2012 19th IEEE International Conference on Image Processing. IEEE, pp 1477\u20131480"},{"issue":"4","key":"18695_CR30","doi-asserted-by":"publisher","first-page":"600","DOI":"10.1109\/TIP.2003.819861","volume":"13","author":"Z Wang","year":"2004","unstructured":"Wang Z, Bovik AC, Sheikh HR, Simoncelli EP (2004) Image quality assessment: From error visibility to structural similarity. IEEE Trans Image Process 13(4):600\u2013612","journal-title":"IEEE Trans Image Process"},{"issue":"12","key":"18695_CR31","doi-asserted-by":"publisher","first-page":"8717","DOI":"10.1109\/TPAMI.2018.2889052","volume":"44","author":"T Afouras","year":"2018","unstructured":"Afouras T, Chung JS, Senior A, Vinyals O, Zisserman A (2018) Deep audio-visual speech recognition. IEEE Trans Pattern Anal Mach Intell 44(12):8717\u20138727. https:\/\/doi.org\/10.1109\/TPAMI.2018.2889052","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"18695_CR32","unstructured":"Heusel M, Ramsauer H, Unterthiner T, Nessler B, Hochreiter S (2017) GANs trained by a two time-scale update rule converge to a local Nash equilibrium. In: proceedings of the 31st Conference on Neural Information Processing Systems (NIPS 2017), USA, pp 6629\u20136640"},{"key":"18695_CR33","unstructured":"Saito S, Roy S (2018) Effects of loss functions and target representations on adversarial robustness. arXiv preprint arXiv:1812.00181.\u00a0Accessed 6 Mar 2020"},{"key":"18695_CR34","first-page":"28648","volume":"34","author":"S Kornblith","year":"2021","unstructured":"Kornblith S, Chen T, Lee H, Norouzi M (2021) Why do better loss functions lead to less transferable features? Adv Neural Inf Process Syst 34:28648\u201328662","journal-title":"Adv Neural Inf Process Syst"},{"key":"18695_CR35","doi-asserted-by":"publisher","first-page":"5679","DOI":"10.1109\/ICASSP.2018.8461852","volume-title":"2018 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","author":"L Juvela","year":"2018","unstructured":"Juvela L, Bollepalli B, Wang X, Kameoka H, Airaksinen M, Yamagishi J, Alku P (2018) Speech waveform synthesis from MFCC sequences with generative adversarial networks. 2018 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP). IEEE, pp 5679\u20135683"},{"key":"18695_CR36","doi-asserted-by":"publisher","unstructured":"Wang K, Wu Q, Song L, Yang Z, Wu W, Qian C, ... Loy CC (2020) Mead: A large-scale audio-visual dataset for emotional talking-face generation. In: European Conference on Computer Vision. Springer, Cham, pp 700\u2013717. https:\/\/doi.org\/10.1007\/978-3-030-58589-1_42","DOI":"10.1007\/978-3-030-58589-1_42"},{"issue":"8","key":"18695_CR37","doi-asserted-by":"publisher","first-page":"1735","DOI":"10.1162\/neco.1997.9.8.1735","volume":"9","author":"J Schmidhuber","year":"1997","unstructured":"Schmidhuber J, Hochreiter S (1997) Long short-term memory. Neural Comput 9(8):1735\u20131780","journal-title":"Neural Comput"}],"container-title":["Multimedia Tools and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11042-024-18695-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11042-024-18695-x\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11042-024-18695-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,11]],"date-time":"2024-10-11T07:07:47Z","timestamp":1728630467000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11042-024-18695-x"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,3,11]]},"references-count":37,"journal-issue":{"issue":"35","published-online":{"date-parts":[[2024,10]]}},"alternative-id":["18695"],"URL":"https:\/\/doi.org\/10.1007\/s11042-024-18695-x","relation":{},"ISSN":["1573-7721"],"issn-type":[{"type":"electronic","value":"1573-7721"}],"subject":[],"published":{"date-parts":[[2024,3,11]]},"assertion":[{"value":"9 October 2022","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 January 2024","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 February 2024","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 March 2024","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"Authors have no conflict and\/or competing interest on this work.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}]}}