{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,7,6]],"date-time":"2025-07-06T04:01:06Z","timestamp":1751774466218,"version":"3.41.0"},"reference-count":66,"publisher":"Institute of Electronics, Information and Communications Engineers (IEICE)","issue":"7","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEICE Trans. Inf. & Syst."],"published-print":{"date-parts":[[2025,7,1]]},"DOI":"10.1587\/transinf.2024edp7164","type":"journal-article","created":{"date-parts":[[2025,1,19]],"date-time":"2025-01-19T22:12:15Z","timestamp":1737324735000},"page":"795-807","source":"Crossref","is-referenced-by-count":0,"title":["Mitigating Gender Bias Amplification in Image Captioning"],"prefix":"10.1587","volume":"E108.D","author":[{"given":"Yusuke","family":"HIROTA","sequence":"first","affiliation":[{"name":"Graduate School of Information Science and Technology, Osaka University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yuta","family":"NAKASHIMA","sequence":"additional","affiliation":[{"name":"Graduate School of Information Science and Technology, Osaka University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Noa","family":"GARCIA","sequence":"additional","affiliation":[{"name":"Graduate School of Information Science and Technology, Osaka University"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"532","reference":[{"key":"1","doi-asserted-by":"crossref","unstructured":"[1] Y. Hirota, Y. Nakashima, and N. Garcia, \u201cModel-agnostic gender debiased image captioning,\u201d CVPR, pp.15191-15200, 2023. 10.1109\/cvpr52729.2023.01458","DOI":"10.1109\/CVPR52729.2023.01458"},{"key":"2","doi-asserted-by":"crossref","unstructured":"[2] P. Anderson, X. He, C. Buehler, D. Teney, M. Johnson, S. Gould, and L. Zhang, \u201cBottom-up and top-down attention for image captioning and visual question answering,\u201d CVPR, pp.6077-6086, 2018. 10.1109\/cvpr.2018.00636","DOI":"10.1109\/CVPR.2018.00636"},{"key":"3","unstructured":"[3] B. Wilson, J. Hoffman, and J. Morgenstern, \u201cPredictive inequity in object detection,\u201d arXiv preprint arXiv:1902.11097, 2019."},{"key":"4","doi-asserted-by":"publisher","unstructured":"[4] X. Shen, Y. Wong, and M. Kankanhalli, \u201cFair representation: Guaranteeing approximate multiple group fairness for unknown tasks,\u201d IEEE Trans. Pattern Anal. Mach. Intell., vol.45, no.1, pp.525-538, 2022. 10.1109\/tpami.2022.3148905","DOI":"10.1109\/TPAMI.2022.3148905"},{"key":"5","unstructured":"[5] T. de Vries, I. Misra, C. Wang, and L. van der Maaten, \u201cDoes object recognition work for everyone?,\u201d CVPR Workshops, 2019."},{"key":"6","doi-asserted-by":"crossref","unstructured":"[6] Z. Khan and Y. Fu, \u201cOne label, one billion faces: Usage and consistency of racial categories in computer vision,\u201d ACM FAccT, pp.587-597, 2021. 10.1145\/3442188.3445920","DOI":"10.1145\/3442188.3445920"},{"key":"7","doi-asserted-by":"crossref","unstructured":"[7] P. Stock and M. Cisse, \u201cConvNets and ImageNet beyond accuracy: Understanding mistakes and uncovering biases,\u201d ECCV, pp.504-519, 2018. 10.1007\/978-3-030-01231-1_31","DOI":"10.1007\/978-3-030-01231-1_31"},{"key":"8","doi-asserted-by":"crossref","unstructured":"[8] M. Wang, W. Deng, J. Hu, X. Tao, and Y. Huang, \u201cRacial faces in the wild: Reducing racial bias by information maximization adaptation network,\u201d ICCV, pp.692-702, 2019. 10.1109\/iccv.2019.00078","DOI":"10.1109\/ICCV.2019.00078"},{"key":"9","doi-asserted-by":"crossref","unstructured":"[9] H. Berg, S. Hall, Y. Bhalgat, H. Kirk, A. Shtedritski, and M. Bain, \u201cA prompt array keeps the bias away: Debiasing vision-language models with adversarial learning,\u201d AACL, pp.806-822, 2022. 10.18653\/v1\/2022.aacl-main.61","DOI":"10.18653\/v1\/2022.aacl-main.61"},{"key":"10","unstructured":"[10] J. Buolamwini and T. Gebru, \u201cGender shades: Intersectional accuracy disparities in commercial gender classification,\u201d ACM FAccT, 2018."},{"key":"11","doi-asserted-by":"crossref","unstructured":"[11] O. Vinyals, A. Toshev, S. Bengio, and D. Erhan, \u201cShow and tell: A neural image caption generator,\u201d CVPR, pp.3156-3164, 2015. 10.1109\/cvpr.2015.7298935","DOI":"10.1109\/CVPR.2015.7298935"},{"key":"12","unstructured":"[12] K. Xu, J.L. Ba, R. Kiros, K. Cho, A. Courville, R. Salakhudinov, R.S. Zemel, and Y. Bengio, \u201cShow, attend and tell: Neural image caption generation with visual attention,\u201d ICML, 2015."},{"key":"13","unstructured":"[13] M. Hall, L. van der Maaten, L. Gustafson, and A. Adcock, \u201cA systematic study of bias amplification,\u201d arXiv preprint arXiv:2201.11706, 2022."},{"key":"14","unstructured":"[14] K. Choi, A. Grover, T. Singh, R. Shu, and S. Ermon, \u201cFair generative modeling via weak supervision,\u201d ICML, 2020."},{"key":"15","unstructured":"[15] K. Leino, E. Black, M. Fredrikson, S. Sen, and A. Datta, \u201cFeature-wise bias amplification,\u201d ICLR, 2019."},{"key":"16","doi-asserted-by":"crossref","unstructured":"[16] T. Srinivasan and Y. Bisk, \u201cWorst of both worlds: Biases compound in pre-trained vision-and-language models,\u201d NAACL workshop, pp.77-85, 2022. 10.18653\/v1\/2022.gebnlp-1.10","DOI":"10.18653\/v1\/2022.gebnlp-1.10"},{"key":"17","unstructured":"[17] D. Zhao, J.T. Andrews, and A. Xiang, \u201cMen also do laundry: Multi-attribute bias amplification,\u201d arXiv preprint arXiv:2210.11924, 2022."},{"key":"18","doi-asserted-by":"crossref","unstructured":"[18] L.A. Hendricks, K. Burns, K. Saenko, T. Darrell, and A. Rohrbach, \u201cWomen also snowboard: Overcoming bias in captioning models,\u201d ECCV, pp.793-811, 2018. 10.1007\/978-3-030-01219-9_47","DOI":"10.1007\/978-3-030-01219-9_47"},{"key":"19","unstructured":"[19] A. Wang and O. Russakovsky, \u201cDirectional bias amplification,\u201d ICML, 2021."},{"key":"20","doi-asserted-by":"crossref","unstructured":"[20] Y. Hirota, Y. Nakashima, and N. Garcia, \u201cQuantifying societal bias amplification in image captioning,\u201d CVPR, pp.13440-13449, 2022. 10.1109\/cvpr52688.2022.01309","DOI":"10.1109\/CVPR52688.2022.01309"},{"key":"21","doi-asserted-by":"crossref","unstructured":"[21] R. Tang, M. Du, Y. Li, Z. Liu, N. Zou, and X. Hu, \u201cMitigating gender bias in captioning systems,\u201d WWW, pp.633-645, 2021. 10.1145\/3442381.3449950","DOI":"10.1145\/3442381.3449950"},{"key":"22","unstructured":"[22] S. Bhargava and D. Forsyth, \u201cExposing and correcting the gender bias in image captioning datasets and models,\u201d arXiv preprint arXiv:1912.00578, 2019."},{"key":"23","doi-asserted-by":"crossref","unstructured":"[23] J. Zhao, T. Wang, M. Yatskar, V. Ordonez, and K.-W. Chang, \u201cMen also like shopping: Reducing gender bias amplification using corpus-level constraints,\u201d EMNLP, pp.2979-2989, 2017. 10.18653\/v1\/d17-1323","DOI":"10.18653\/v1\/D17-1323"},{"key":"24","doi-asserted-by":"crossref","unstructured":"[24] K. Papineni, S. Roukos, T. Ward, and W.-J. Zhu, \u201cBLEU: a method for automatic evaluation of machine translation,\u201d ACL, pp.311-318, 2002. 10.3115\/1073083.1073135","DOI":"10.3115\/1073083.1073135"},{"key":"25","doi-asserted-by":"crossref","unstructured":"[25] P. Anderson, B. Fernando, M. Johnson, and S. Gould, \u201cSpice: Semantic propositional image caption evaluation,\u201d ECCV, Springer, pp.382-398, 2016. 10.1007\/978-3-319-46454-1_24","DOI":"10.1007\/978-3-319-46454-1_24"},{"key":"26","doi-asserted-by":"crossref","unstructured":"[26] D. Zhao, A. Wang, and O. Russakovsky, \u201cUnderstanding and evaluating racial biases in image captioning,\u201d ICCV, pp.14810-14820, 2021. 10.1109\/iccv48922.2021.01456","DOI":"10.1109\/ICCV48922.2021.01456"},{"key":"27","doi-asserted-by":"crossref","unstructured":"[27] K. Yang, K. Qinami, L. Fei-Fei, J. Deng, and O. Russakovsky, \u201cTowards fairer datasets: Filtering and balancing the distribution of the people subtree in the ImageNet hierarchy,\u201d ACM FAccT, pp.547-558, 2020. 10.1145\/3351095.3375709","DOI":"10.1145\/3351095.3375709"},{"key":"28","doi-asserted-by":"crossref","unstructured":"[28] C. Ross, B. Katz, and A. Barbu, \u201cMeasuring social biases in grounded vision and language embeddings,\u201d NAACL-HLT, pp.998-1008, 2021. 10.18653\/v1\/2021.naacl-main.78","DOI":"10.18653\/v1\/2021.naacl-main.78"},{"key":"29","doi-asserted-by":"crossref","unstructured":"[29] N. Garcia, Y. Hirota, W. Yankun, and Y. Nakashima, \u201cUncurated image-text datasets: Shedding light on demographic bias,\u201d CVPR, pp.6957-6966, 2023. 10.1109\/cvpr52729.2023.00672","DOI":"10.1109\/CVPR52729.2023.00672"},{"key":"30","doi-asserted-by":"crossref","unstructured":"[30] T.-Y. Lin, M. Maire, S. Belongie, J. Hays, P. Perona, D. Ramanan, P. Doll\u00e1r, and C.L. Zitnick, \u201cMicrosoft COCO: Common objects in context,\u201d ECCV, pp.740-755, 2014. 10.1007\/978-3-319-10602-1_48","DOI":"10.1007\/978-3-319-10602-1_48"},{"key":"31","doi-asserted-by":"crossref","unstructured":"[31] T. Wang, J. Zhao, M. Yatskar, K.-W. Chang, and V. Ordonez, \u201cBalanced datasets are not enough: Estimating and mitigating gender bias in deep image representations,\u201d ICCV, pp.5309-5318, 2019. 10.1109\/iccv.2019.00541","DOI":"10.1109\/ICCV.2019.00541"},{"key":"32","unstructured":"[32] N. Jain, A. Olmo, S. Sengupta, L. Manikonda, and S. Kambhampati, \u201cImperfect imaganation: Implications of gans exacerbating biases on facial data augmentation and snapchat selfie lenses,\u201d arXiv preprint arXiv:2001.09528, 2020."},{"key":"33","doi-asserted-by":"crossref","unstructured":"[33] S. Jia, T. Meng, J. Zhao, and K.-W. Chang, \u201cMitigating gender bias amplification in distribution by posterior regularization,\u201d ACL, pp.2936-2942, 2020. 10.18653\/v1\/2020.acl-main.264","DOI":"10.18653\/v1\/2020.acl-main.264"},{"key":"34","doi-asserted-by":"crossref","unstructured":"[34] Y. Zhang, J. Wang, and J. Sang, \u201cCounterfactually measuring and eliminating social bias in vision-language pre-training models,\u201d ACMMM, pp.4996-5004, 2022. 10.1145\/3503161.3548396","DOI":"10.1145\/3503161.3548396"},{"key":"35","doi-asserted-by":"crossref","unstructured":"[35] S. Whitehead, S. Petryk, V. Shakib, J. Gonzalez, T. Darrell, A. Rohrbach, and M. Rohrbach, \u201cReliable visual question answering: Abstain rather than answer incorrectly,\u201d ECCV, pp.148-166, 2022. 10.1007\/978-3-031-20059-5_9","DOI":"10.1007\/978-3-031-20059-5_9"},{"key":"36","doi-asserted-by":"crossref","unstructured":"[36] Z. Wang, K. Qinami, I.C. Karakozis, K. Genova, P. Nair, K. Hata, and O. Russakovsky, \u201cTowards fairness in visual recognition: Effective strategies for bias mitigation,\u201d CVPR, pp.8916-8925, 2020. 10.1109\/cvpr42600.2020.00894","DOI":"10.1109\/CVPR42600.2020.00894"},{"key":"37","doi-asserted-by":"crossref","unstructured":"[37] W. Thong and C.G. Snoek, \u201cFeature and label embedding spaces matter in addressing image classifier bias,\u201d BMVC, 2021.","DOI":"10.5244\/C.35.46"},{"key":"38","doi-asserted-by":"crossref","unstructured":"[38] A. Wang, A. Narayanan, and O. Russakovsky, \u201cRevise: A tool for measuring and mitigating bias in visual datasets,\u201d ECCV, pp.733-751, 2020. 10.1007\/978-3-030-58580-8_43","DOI":"10.1007\/978-3-030-58580-8_43"},{"key":"39","unstructured":"[39] R. Yao, Z. Cui, X. Li, and L. Gu, \u201cImproving fairness in image classification via sketching,\u201d NeurIPS Workshop, 2022."},{"key":"40","doi-asserted-by":"publisher","unstructured":"[40] W. Rawat and Z. Wang, \u201cDeep convolutional neural networks for image classification: A comprehensive review,\u201d Neural computation, vol.29, no.9, pp.2352-2449, 2017. 10.1162\/neco_a_00990","DOI":"10.1162\/neco_a_00990"},{"key":"41","doi-asserted-by":"crossref","unstructured":"[41] M. Yatskar, L. Zettlemoyer, and A. Farhadi, \u201cSituation recognition: Visual semantic role labeling for image understanding,\u201d CVPR, pp.5534-5542, 2016. 10.1109\/cvpr.2016.597","DOI":"10.1109\/CVPR.2016.597"},{"key":"42","doi-asserted-by":"crossref","unstructured":"[42] F. Sammani and L. Melas-Kyriazi, \u201cShow, edit and tell: a framework for editing image captions,\u201d CVPR, pp.4807-4815, 2020. 10.1109\/cvpr42600.2020.00486","DOI":"10.1109\/CVPR42600.2020.00486"},{"key":"43","unstructured":"[43] F. Sammani and M. Elsayed, \u201cLook and modify: Modification networks for image captioning,\u201d BMVC, 2019."},{"key":"44","doi-asserted-by":"crossref","unstructured":"[44] Z. Wang, L. Chen, W. Ma, G. Han, Y. Niu, J. Shao, and J. Xiao, \u201cExplicit image caption editing,\u201d ECCV, pp.113-129, 2022. 10.1007\/978-3-031-20059-5_7","DOI":"10.1007\/978-3-031-20059-5_7"},{"key":"45","doi-asserted-by":"crossref","unstructured":"[45] H.R. Tavakoli, R. Shetty, A. Borji, and J. Laaksonen, \u201cPaying attention to descriptions generated by image captioning models,\u201d ICCV, pp.2506-2515, 2017. 10.1109\/iccv.2017.272","DOI":"10.1109\/ICCV.2017.272"},{"key":"46","doi-asserted-by":"crossref","unstructured":"[46] A.C. Berg, T.L. Berg, H. Daum\u00e9, J. Dodge, A. Goyal, X. Han, A. Mensch, M. Mitchell, A. Sood, K. Stratos, and K. Yamaguchi, \u201cUnderstanding and predicting importance in images,\u201d CVPR, IEEE, pp.3562-3569, 2012. 10.1109\/cvpr.2012.6248100","DOI":"10.1109\/CVPR.2012.6248100"},{"key":"47","unstructured":"[47] C. Raffel, N. Shazeer, A. Roberts, K. Lee, S. Narang, M. Matena, Y. Zhou, W. Li, P.J. Liu, et al., \u201cExploring the limits of transfer learning with a unified text-to-text transformer.,\u201d J. Mach. Learn. Res., 2020."},{"key":"48","unstructured":"[48] J. Devlin, M.-W. Chang, K. Lee, and K. Toutanova, \u201cBERT: pre-training of deep bidirectional transformers for language understanding,\u201d NAACL-HLT, vol.1, pp.4171-4186, 2019. 10.18653\/v1\/n19-1423"},{"key":"49","doi-asserted-by":"publisher","unstructured":"[49] G.A. Miller, \u201cWordnet: a lexical database for english,\u201d Communications of the ACM, vol.38, no.11, pp.39-41, 1995. 10.1145\/219717.219748","DOI":"10.1145\/219717.219748"},{"key":"50","unstructured":"[50] W. Kim, B. Son, and I. Kim, \u201cVilt: Vision-and-language transformer without convolution or region supervision,\u201d ICLR, PMLR, 2021."},{"key":"51","unstructured":"[51] A. Radford, J. Wu, R. Child, D. Luan, D. Amodei, and I. Sutskever, \u201cLanguage models are unsupervised multitask learners,\u201d OpenAI blog, 2019."},{"key":"52","doi-asserted-by":"publisher","unstructured":"[52] S. Rothe, S. Narayan, and A. Severyn, \u201cLeveraging pre-trained checkpoints for sequence generation tasks,\u201d Transactions of the Association for Computational Linguistics, vol.8, pp.264-280, 2020. 10.1162\/tacl_a_00313","DOI":"10.1162\/tacl_a_00313"},{"key":"53","unstructured":"[53] A. Radford, J.W. Kim, C. Hallacy, A. Ramesh, G. Goh, S. Agarwal, G. Sastry, A. Askell, P. Mishkin, J. Clark, et al., \u201cLearning transferable visual models from natural language supervision,\u201d ICML, 2021."},{"key":"54","doi-asserted-by":"crossref","unstructured":"[54] S.J. Rennie, E. Marcheret, Y. Mroueh, J. Ross, and V. Goel, \u201cSelf-critical sequence training for image captioning,\u201d CVPR, pp.1179-1195, 2017. 10.1109\/cvpr.2017.131","DOI":"10.1109\/CVPR.2017.131"},{"key":"55","unstructured":"[55] A. Vaswani, N. Shazeer, N. Parmar, J. Uszkoreit, L. Jones, A.N. Gomez, \u0141. Kaiser, and I. Polosukhin, \u201cAttention is all you need,\u201d NeurIPS, 2017."},{"key":"56","doi-asserted-by":"crossref","unstructured":"[56] X. Li, X. Yin, C. Li, P. Zhang, X. Hu, L. Zhang, L. Wang, H. Hu, L. Dong, F. Wei, Y. Choi, and J. Gao, \u201cOscar: Object-semantics aligned pre-training for vision-language tasks,\u201d ECCV, pp.121-137, 2020. 10.1007\/978-3-030-58577-8_8","DOI":"10.1007\/978-3-030-58577-8_8"},{"key":"57","unstructured":"[57] R. Mokady, A. Hertz, and A.H. Bermano, \u201cClipcap: Clip prefix for image captioning,\u201d arXiv preprint arXiv:2111.09734, 2021."},{"key":"58","doi-asserted-by":"crossref","unstructured":"[58] V.-Q. Nguyen, M. Suganuma, and T. Okatani, \u201cGrit: Faster and better image captioning transformer using dual visual features,\u201d ECCV, Springer, pp.167-184, 2022. 10.1007\/978-3-031-20059-5_10","DOI":"10.1007\/978-3-031-20059-5_10"},{"key":"59","unstructured":"[59] X. Chen, H. Fang, T.Y. Lin, R. Vedantam, S. Gupta, P. Doll\u00e1r, and C.L. Zitnick, \u201cMicrosoft COCO captions: Data collection and evaluation server,\u201d arXiv preprint arXiv:1504.00325, 2015."},{"key":"60","doi-asserted-by":"crossref","unstructured":"[60] R. Vedantam, C. Lawrence Zitnick, and D. Parikh, \u201cCIDEr: Consensus-based image description evaluation,\u201d CVPR, pp.4566-4575, 2015. 10.1109\/cvpr.2015.7299087","DOI":"10.1109\/CVPR.2015.7299087"},{"key":"61","doi-asserted-by":"crossref","unstructured":"[61] M. Denkowski and A. Lavie, \u201cMeteor universal: Language specific translation evaluation for any target language,\u201d Workshop on Statistical Machine Translation, pp.376-380, 2014. 10.3115\/v1\/w14-3348","DOI":"10.3115\/v1\/W14-3348"},{"key":"62","doi-asserted-by":"crossref","unstructured":"[62] J. Hessel, A. Holtzman, M. Forbes, R.L. Bras, and Y. Choi, \u201cClipscore: A reference-free evaluation metric for image captioning,\u201d EMNLP (1), pp.7514-7528, 2021. 10.18653\/v1\/2021.emnlp-main.595","DOI":"10.18653\/v1\/2021.emnlp-main.595"},{"key":"63","doi-asserted-by":"crossref","unstructured":"[63] A. Karpathy and L. Fei-Fei, \u201cDeep visual-semantic alignments for generating image descriptions,\u201d CVPR, pp.3128-3137, 2015. 10.1109\/cvpr.2015.7298932","DOI":"10.1109\/CVPR.2015.7298932"},{"key":"64","doi-asserted-by":"publisher","unstructured":"[64] S. Ren, K. He, R. Girshick, and J. Sun, \u201cFaster r-cnn: towards real-time object detection with region proposal networks,\u201d IEEE Trans. Pattern Anal. Mach. Intell., vol.39, no.6, pp.1137-1149, 2016. 10.1109\/tpami.2016.2577031","DOI":"10.1109\/TPAMI.2016.2577031"},{"key":"65","unstructured":"[65] J. Li, D. Li, S. Savarese, and S. Hoi, \u201cBlip-2: Bootstrapping language-image pre-training with frozen image encoders and large language models,\u201d ICML, 2023."},{"key":"66","doi-asserted-by":"crossref","unstructured":"[66] E. Loper and S. Bird, \u201cNltk: The natural language toolkit,\u201d arXiv preprint cs\/0205028, 2002.","DOI":"10.3115\/1118108.1118117"}],"container-title":["IEICE Transactions on Information and Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.jstage.jst.go.jp\/article\/transinf\/E108.D\/7\/E108.D_2024EDP7164\/_pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,5]],"date-time":"2025-07-05T03:36:01Z","timestamp":1751686561000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.jstage.jst.go.jp\/article\/transinf\/E108.D\/7\/E108.D_2024EDP7164\/_article"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,1]]},"references-count":66,"journal-issue":{"issue":"7","published-print":{"date-parts":[[2025]]}},"URL":"https:\/\/doi.org\/10.1587\/transinf.2024edp7164","relation":{},"ISSN":["0916-8532","1745-1361"],"issn-type":[{"type":"print","value":"0916-8532"},{"type":"electronic","value":"1745-1361"}],"subject":[],"published":{"date-parts":[[2025,7,1]]},"article-number":"2024EDP7164"}}