{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,7,6]],"date-time":"2025-07-06T04:01:06Z","timestamp":1751774466227,"version":"3.41.0"},"reference-count":53,"publisher":"Institute of Electronics, Information and Communications Engineers (IEICE)","issue":"7","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEICE Trans. Inf. & Syst."],"published-print":{"date-parts":[[2025,7,1]]},"DOI":"10.1587\/transinf.2024edp7116","type":"journal-article","created":{"date-parts":[[2025,1,19]],"date-time":"2025-01-19T22:12:12Z","timestamp":1737324732000},"page":"784-794","source":"Crossref","is-referenced-by-count":0,"title":["Societal Bias in Image Captioning: Identifying and Measuring Bias Amplification"],"prefix":"10.1587","volume":"E108.D","author":[{"given":"Yusuke","family":"HIROTA","sequence":"first","affiliation":[{"name":"Graduate School of Information Science and Technology, Osaka University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yuta","family":"NAKASHIMA","sequence":"additional","affiliation":[{"name":"Graduate School of Information Science and Technology, Osaka University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Noa","family":"GARCIA","sequence":"additional","affiliation":[{"name":"Graduate School of Information Science and Technology, Osaka University"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"532","reference":[{"key":"1","doi-asserted-by":"crossref","unstructured":"[1] Y. Hirota, Y. Nakashima, and N. Garcia, \u201cQuantifying societal bias amplification in image captioning,\u201d CVPR, pp.13440-13449, 2022. 10.1109\/cvpr52688.2022.01309","DOI":"10.1109\/CVPR52688.2022.01309"},{"key":"2","doi-asserted-by":"crossref","unstructured":"[2] N. Garcia, Y. Hirota, Y. Wu, and Y. Nakashima, \u201cUncurated image-text datasets: Shedding light on demographic bias,\u201d CVPR, pp.6957-6966, 2023. 10.1109\/cvpr52729.2023.00672","DOI":"10.1109\/CVPR52729.2023.00672"},{"key":"3","unstructured":"[3] J. Buolamwini and T. Gebru, \u201cGender shades: Intersectional accuracy disparities in commercial gender classification,\u201d ACM FAccT, 2018."},{"key":"4","doi-asserted-by":"crossref","unstructured":"[4] J. Zhao, T. Wang, M. Yatskar, V. Ordonez, and K.-W. Chang, \u201cMen also like shopping: Reducing gender bias amplification using corpus-level constraints,\u201d EMNLP, pp.2979-2989, 2017. 10.18653\/v1\/d17-1323","DOI":"10.18653\/v1\/D17-1323"},{"key":"5","doi-asserted-by":"crossref","unstructured":"[5] L.A. Hendricks, K. Burns, K. Saenko, T. Darrell, and A. Rohrbach, \u201cWomen also snowboard: Overcoming bias in captioning models,\u201d ECCV, pp.793-811, 2018. 10.1007\/978-3-030-01219-9_47","DOI":"10.1007\/978-3-030-01219-9_47"},{"key":"6","doi-asserted-by":"crossref","unstructured":"[6] A. Wang, A. Narayanan, and O. Russakovsky, \u201cRevise: A tool for measuring and mitigating bias in visual datasets,\u201d ECCV, pp.733-751, 2020. 10.1007\/978-3-030-58580-8_43","DOI":"10.1007\/978-3-030-58580-8_43"},{"key":"7","doi-asserted-by":"crossref","unstructured":"[7] K. Yang, K. Qinami, L. Fei-Fei, J. Deng, and O. Russakovsky, \u201cTowards fairer datasets: Filtering and balancing the distribution of the people subtree in the ImageNet hierarchy,\u201d ACM FAccT, pp.547-558, 2020. 10.1145\/3351095.3375709","DOI":"10.1145\/3351095.3375709"},{"key":"8","doi-asserted-by":"crossref","unstructured":"[8] Z. Khan and Y. Fu, \u201cOne label, one billion faces: Usage and consistency of racial categories in computer vision,\u201d ACM FAccT, pp.587-597, 2021. 10.1145\/3442188.3445920","DOI":"10.1145\/3442188.3445920"},{"key":"9","unstructured":"[9] T. de Vries, I. Misra, C. Wang, and L. van der Maaten, \u201cDoes object recognition work for everyone?,\u201d CVPR Workshops, 2019."},{"key":"10","doi-asserted-by":"crossref","unstructured":"[10] P. Stock and M. Cisse, \u201cConvNets and ImageNet beyond accuracy: Understanding mistakes and uncovering biases,\u201d ECCV, pp.504-519, 2018. 10.1007\/978-3-030-01231-1_31","DOI":"10.1007\/978-3-030-01231-1_31"},{"key":"11","doi-asserted-by":"crossref","unstructured":"[11] M. Alvi, A. Zisserman, and C. Nell\u00e5ker, \u201cTurning a blind eye: Explicit removal of biases and variation from deep neural network embeddings,\u201d ECCV Workshops, 2018. 10.1007\/978-3-030-11009-3_34","DOI":"10.1007\/978-3-030-11009-3_34"},{"key":"12","doi-asserted-by":"crossref","unstructured":"[12] C. D\u2019ignazio and L.F. Klein, Data feminism, MIT press, 2020.","DOI":"10.7551\/mitpress\/11805.001.0001"},{"key":"13","doi-asserted-by":"crossref","unstructured":"[13] T. Wang, J. Zhao, M. Yatskar, K.-W. Chang, and V. Ordonez, \u201cBalanced datasets are not enough: Estimating and mitigating gender bias in deep image representations,\u201d ICCV, pp.5309-5318, 2019. 10.1109\/iccv.2019.00541","DOI":"10.1109\/ICCV.2019.00541"},{"key":"14","unstructured":"[14] X. Chen, H. Fang, T.Y. Lin, R. Vedantam, S. Gupta, P. Doll\u00e1r, and C.L. Zitnick, \u201cMicrosoft COCO captions: Data collection and evaluation server,\u201d arXiv preprint arXiv:1504.00325, 2015."},{"key":"15","doi-asserted-by":"crossref","unstructured":"[15] O. Vinyals, A. Toshev, S. Bengio, and D. Erhan, \u201cShow and tell: A neural image caption generator,\u201d CVPR, pp.3156-3164, 2015. 10.1109\/cvpr.2015.7298935","DOI":"10.1109\/CVPR.2015.7298935"},{"key":"16","doi-asserted-by":"crossref","unstructured":"[16] A. Torralba and A.A. Efros, \u201cUnbiased look at dataset bias,\u201d CVPR, pp.1521-1528, 2011. 10.1109\/cvpr.2011.5995347","DOI":"10.1109\/CVPR.2011.5995347"},{"key":"17","unstructured":"[17] B. Wilson, J. Hoffman, and J. Morgenstern, \u201cPredictive inequity in object detection,\u201d arXiv preprint arXiv:1902.11097, 2019."},{"key":"18","doi-asserted-by":"crossref","unstructured":"[18] X. Li, X. Yin, C. Li, P. Zhang, X. Hu, L. Zhang, L. Wang, H. Hu, L. Dong, F. Wei, Y. Choi, and J. Gao, \u201cOscar: Object-semantics aligned pre-training for vision-language tasks,\u201d ECCV, pp.121-137, 2020. 10.1007\/978-3-030-58577-8_8","DOI":"10.1007\/978-3-030-58577-8_8"},{"key":"19","doi-asserted-by":"crossref","unstructured":"[19] P. Sharma, N. Ding, S. Goodman, and R. Soricut, \u201cConceptual captions: A cleaned, hypernymed, image alt-text dataset for automatic image captioning,\u201d ACL, pp.2556-2565, 2018. 10.18653\/v1\/p18-1238","DOI":"10.18653\/v1\/P18-1238"},{"key":"20","unstructured":"[20] A. Vaswani, N. Shazeer, N. Parmar, J. Uszkoreit, L. Jones, A.N. Gomez, \u0141. Kaiser, and I. Polosukhin, \u201cAttention is all you need,\u201d NeurIPS, 2017."},{"key":"21","unstructured":"[21] A. Birhane, V.U. Prabhu, and E. Kahembwe, \u201cMultimodal datasets: Misogyny, pornography, and malignant stereotypes,\u201d arXiv preprint arXiv:2110.01963, 2021."},{"key":"22","doi-asserted-by":"crossref","unstructured":"[22] D. Zhao, A. Wang, and O. Russakovsky, \u201cUnderstanding and evaluating racial biases in image captioning,\u201d ICCV, pp.14810-14820, 2021. 10.1109\/iccv48922.2021.01456","DOI":"10.1109\/ICCV48922.2021.01456"},{"key":"23","doi-asserted-by":"crossref","unstructured":"[23] R. Tang, M. Du, Y. Li, Z. Liu, N. Zou, and X. Hu, \u201cMitigating gender bias in captioning systems,\u201d WWW, pp.633-645, 2021. 10.1145\/3442381.3449950","DOI":"10.1145\/3442381.3449950"},{"key":"24","unstructured":"[24] L. Fei-Fei, R. Fergus, and P. Perona, \u201cLearning generative visual models from few training examples: An incremental bayesian approach tested on 101 object categories,\u201d CVPR Workshops, pp.178-178, 2004. 10.1109\/cvpr.2004.383"},{"key":"25","doi-asserted-by":"crossref","unstructured":"[25] J. Deng, W. Dong, R. Socher, L.-J. Li, K. Li, and L. Fei-Fei, \u201cImageNet: A large-scale hierarchical image database,\u201d CVPR, pp.248-255, 2009. 10.1109\/cvpr.2009.5206848","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"26","doi-asserted-by":"crossref","unstructured":"[26] T.-Y. Lin, M. Maire, S. Belongie, J. Hays, P. Perona, D. Ramanan, P. Doll\u00e1r, and C.L. Zitnick, \u201cMicrosoft COCO: Common objects in context,\u201d ECCV, pp.740-755, 2014. 10.1007\/978-3-319-10602-1_48","DOI":"10.1007\/978-3-319-10602-1_48"},{"key":"27","doi-asserted-by":"crossref","unstructured":"[27] M. Wang, W. Deng, J. Hu, X. Tao, and Y. Huang, \u201cRacial faces in the wild: Reducing racial bias by information maximization adaptation network,\u201d ICCV, pp.692-702, 2019. 10.1109\/iccv.2019.00078","DOI":"10.1109\/ICCV.2019.00078"},{"key":"28","unstructured":"[28] A. Wang and O. Russakovsky, \u201cDirectional bias amplification,\u201d ICML, 2021."},{"key":"29","doi-asserted-by":"crossref","unstructured":"[29] Z. Wang, K. Qinami, I.C. Karakozis, K. Genova, P. Nair, K. Hata, and O. Russakovsky, \u201cTowards fairness in visual recognition: Effective strategies for bias mitigation,\u201d CVPR, pp.8916-8925, 2020. 10.1109\/cvpr42600.2020.00894","DOI":"10.1109\/CVPR42600.2020.00894"},{"key":"30","unstructured":"[30] K. Xu, J. Ba, R. Kiros, K. Cho, A. Courville, R. Salakhudinov, R. Zemel, and Y. Bengio, \u201cShow, attend and tell: Neural image caption generation with visual attention,\u201d ICML, 2015."},{"key":"31","doi-asserted-by":"crossref","unstructured":"[31] Q. You, H. Jin, Z. Wang, C. Fang, and J. Luo, \u201cImage captioning with semantic attention,\u201d CVPR, pp.4651-4659, 2016. 10.1109\/cvpr.2016.503","DOI":"10.1109\/CVPR.2016.503"},{"key":"32","doi-asserted-by":"crossref","unstructured":"[32] P. Anderson, X. He, C. Buehler, D. Teney, M. Johnson, S. Gould, and L. Zhang, \u201cBottom-up and top-down attention for image captioning and visual question answering,\u201d CVPR, pp.6077-6086, 2018. 10.1109\/cvpr.2018.00636","DOI":"10.1109\/CVPR.2018.00636"},{"key":"33","doi-asserted-by":"publisher","unstructured":"[33] E. Abdelrahman, P. Sun, L.E. Li, and M. Elhoseiny, \u201cImagecaptioner2: Image captioner for image captioning bias amplification assessment,\u201d AAAI, vol.38, no.19, pp.20902-20911, 2024. 10.1609\/aaai.v38i19.30080","DOI":"10.1609\/aaai.v38i19.30080"},{"key":"34","doi-asserted-by":"crossref","unstructured":"[34] K. Papineni, S. Roukos, T. Ward, and W.-J. Zhu, \u201cBLEU: a method for automatic evaluation of machine translation,\u201d ACL, pp.311-318, 2002. 10.3115\/1073083.1073135","DOI":"10.3115\/1073083.1073135"},{"key":"35","doi-asserted-by":"crossref","unstructured":"[35] R. Vedantam, C.L. Zitnick, and D. Parikh, \u201cCIDEr: Consensus-based image description evaluation,\u201d CVPR, pp.4566-4575, 2015. 10.1109\/cvpr.2015.7299087","DOI":"10.1109\/CVPR.2015.7299087"},{"key":"36","doi-asserted-by":"crossref","unstructured":"[36] M. Denkowski and A. Lavie, \u201cMeteor universal: Language specific translation evaluation for any target language,\u201d Workshop on statistical machine translation, pp.376-380, 2014. 10.3115\/v1\/w14-3348","DOI":"10.3115\/v1\/W14-3348"},{"key":"37","unstructured":"[37] C.-Y. Lin, \u201cRouge: A package for automatic evaluation of summaries,\u201d Text summarization branches out, 2004."},{"key":"38","doi-asserted-by":"crossref","unstructured":"[38] S.J. Rennie, E. Marcheret, Y. Mroueh, J. Ross, and V. Goel, \u201cSelf-critical sequence training for image captioning,\u201d CVPR, pp.1179-1195, 2017. 10.1109\/cvpr.2017.131","DOI":"10.1109\/CVPR.2017.131"},{"key":"39","doi-asserted-by":"crossref","unstructured":"[39] Y. LeCun, Y. Bengio, and G. Hinton, \u201cDeep learning,\u201d Nature, vol.521, no.7553, pp.436-444, 2015. 10.1038\/nature14539","DOI":"10.1038\/nature14539"},{"key":"40","doi-asserted-by":"crossref","unstructured":"[40] S. Hochreiter and J. Schmidhuber, \u201cLong short-term memory,\u201d Neural computation, vol.9, no.8, pp.1735-1780, 1997. 10.1162\/neco.1997.9.8.1735","DOI":"10.1162\/neco.1997.9.8.1735"},{"key":"41","doi-asserted-by":"crossref","unstructured":"[41] A. Karpathy and L. Fei-Fei, \u201cDeep visual-semantic alignments for generating image descriptions,\u201d CVPR, pp.3128-3137, 2015. 10.1109\/cvpr.2015.7298932","DOI":"10.1109\/CVPR.2015.7298932"},{"key":"42","unstructured":"[42] T. Bolukbasi, K.-W. Chang, J.Y. Zou, V. Saligrama, and A.T. Kalai, \u201cMan is to computer programmer as woman is to homemaker? debiasing word embeddings,\u201d NeurIPS, 2016."},{"key":"43","unstructured":"[43] S. Dev and J. Phillips, \u201cAttenuating bias in word vectors,\u201d International Conference on Artificial Intelligence and Statistics, PMLR, 2019."},{"key":"44","unstructured":"[44] J. Devlin, M.-W. Chang, K. Lee, and K. Toutanova, \u201cBERT: pre-training of deep bidirectional transformers for language understanding,\u201d NAACL-HLT, vol.1, pp.4171-4186, 2019. 10.18653\/v1\/n19-1423"},{"key":"45","doi-asserted-by":"publisher","unstructured":"[45] R. Bhardwaj, N. Majumder, and S. Poria, \u201cInvestigating gender bias in BERT,\u201d Cognitive Computation, vol.13, no.4, pp.1008-1018, 2021. 10.1007\/s12559-021-09881-2","DOI":"10.1007\/s12559-021-09881-2"},{"key":"46","doi-asserted-by":"crossref","unstructured":"[46] E.M. Bender, T. Gebru, A. McMillan-Major, and S. Shmitchell, \u201cOn the dangers of stochastic parrots: Can language models be too big?,\u201d ACM FAccT, pp.610-623, 2021. 10.1145\/3442188.3445922","DOI":"10.1145\/3442188.3445922"},{"key":"47","doi-asserted-by":"crossref","unstructured":"[47] N. Meister, D. Zhao, A. Wang, V.V. Ramaswamy, R. Fong, and O. Russakovsky, \u201cGender artifacts in visual datasets,\u201d ICCV, pp.4814-4825, 2023. 10.1109\/iccv51070.2023.00446","DOI":"10.1109\/ICCV51070.2023.00446"},{"key":"48","unstructured":"[48] P. Wang, A. Yang, R. Men, J. Lin, S. Bai, Z. Li, J. Ma, C. Zhou, J. Zhou, and H. Yang, \u201cOfa: Unifying architectures, tasks, and modalities through a simple sequence-to-sequence learning framework,\u201d ICML, 2022."},{"key":"49","unstructured":"[49] R. Mokady, A. Hertz, and A.H. Bermano, \u201cClipcap: Clip prefix for image captioning,\u201d arXiv preprint arXiv:2111.09734, 2021."},{"key":"50","doi-asserted-by":"crossref","unstructured":"[50] H. Jiang, I. Misra, M. Rohrbach, E. Learned-Miller, and X. Chen, \u201cIn defense of grid features for visual question answering,\u201d CVPR, pp.10267-10276, 2020. 10.1109\/cvpr42600.2020.01028","DOI":"10.1109\/CVPR42600.2020.01028"},{"key":"51","doi-asserted-by":"crossref","unstructured":"[51] K. He, X. Zhang, S. Ren, and J. Sun, \u201cDeep residual learning for image recognition,\u201d CVPR, pp.770-778, 2016. 10.1109\/cvpr.2016.90","DOI":"10.1109\/CVPR.2016.90"},{"key":"52","doi-asserted-by":"publisher","unstructured":"[52] S. Ren, K. He, R. Girshick, and J. Sun, \u201cFaster r-cnn: towards real-time object detection with region proposal networks,\u201d IEEE Trans. Pattern Anal. Mach. Intell., vol.39, no.6, pp.1137-1149, 2016. 10.1109\/tpami.2016.2577031","DOI":"10.1109\/TPAMI.2016.2577031"},{"key":"53","doi-asserted-by":"crossref","unstructured":"[53] Y. Hirota, Y. Nakashima, and N. Garcia, \u201cModel-agnostic gender debiased image captioning,\u201d CVPR, pp.15191-15200, 2023. 10.1109\/cvpr52729.2023.01458","DOI":"10.1109\/CVPR52729.2023.01458"}],"container-title":["IEICE Transactions on Information and Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.jstage.jst.go.jp\/article\/transinf\/E108.D\/7\/E108.D_2024EDP7116\/_pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,5]],"date-time":"2025-07-05T03:35:52Z","timestamp":1751686552000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.jstage.jst.go.jp\/article\/transinf\/E108.D\/7\/E108.D_2024EDP7116\/_article"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,1]]},"references-count":53,"journal-issue":{"issue":"7","published-print":{"date-parts":[[2025]]}},"URL":"https:\/\/doi.org\/10.1587\/transinf.2024edp7116","relation":{},"ISSN":["0916-8532","1745-1361"],"issn-type":[{"type":"print","value":"0916-8532"},{"type":"electronic","value":"1745-1361"}],"subject":[],"published":{"date-parts":[[2025,7,1]]},"article-number":"2024EDP7116"}}