{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,2]],"date-time":"2026-06-02T17:26:10Z","timestamp":1780421170502,"version":"3.54.1"},"reference-count":49,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Open J. Comput. Soc."],"published-print":{"date-parts":[[2025]]},"DOI":"10.1109\/ojcs.2025.3560333","type":"journal-article","created":{"date-parts":[[2025,4,14]],"date-time":"2025-04-14T17:41:36Z","timestamp":1744652496000},"page":"658-668","source":"Crossref","is-referenced-by-count":6,"title":["A Detailed Comparative Analysis of Automatic Neural Metrics for Machine Translation: BLEURT & BERTScore"],"prefix":"10.1109","volume":"6","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-1273-8558","authenticated-orcid":false,"given":"Aniruddha","family":"Mukherjee","sequence":"first","affiliation":[{"name":"School of Computer Engineering, Kalinga Institute of Industrial Technology (KIIT) Deemed to be University, Bhubaneswar, Odisha, India"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-9023-1661","authenticated-orcid":false,"given":"Vikas","family":"Hassija","sequence":"additional","affiliation":[{"name":"School of Computer Engineering, Kalinga Institute of Industrial Technology (KIIT) Deemed to be University, Bhubaneswar, Odisha, India"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6730-3060","authenticated-orcid":false,"given":"Vinay","family":"Chamola","sequence":"additional","affiliation":[{"name":"Department of Electrical and Electronics Engineering, Birla Institute of Technology and Science, Pilani, Pilani Campus, Vidya Vihar, Pilani, Rajasthan, India"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0003-4601","authenticated-orcid":false,"given":"Karunesh Kumar","family":"Gupta","sequence":"additional","affiliation":[{"name":"Department of Electrical and Electronics Engineering, Birla Institute of Technology and Science, Pilani, Pilani Campus, Vidya Vihar, Pilani, Rajasthan, India"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1016\/S0065-2458(08)60607-5"},{"issue":"2","key":"ref2","first-page":"263","article-title":"The mathematics of statistical machine translation: Parameter estimation","volume":"19","author":"Brown","year":"1993","journal-title":"Comput. Linguistics"},{"key":"ref3","article-title":"Sequence to sequence learning with neural networks","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"27","author":"Sutskever","year":"2014"},{"key":"ref4","article-title":"Neural machine translation by jointly learning to align and translate","author":"Bahdanau"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/MCI.2018.2840738"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1126\/science.aaa8685"},{"key":"ref8","first-page":"2376","article-title":"A study of automatic metrics for the evaluation of natural language explanations","volume-title":"Proc. 16th Conf. Eur. Chapter Assoc. Comput. Linguistics","author":"Clinciu","year":"2021"},{"key":"ref9","doi-asserted-by":"crossref","first-page":"180","DOI":"10.18653\/v1\/2022.insights-1.25","article-title":"Pre-trained language models evaluating themselvesa comparative study","volume-title":"Proc. 3rd Workshop Insights Negative Results","author":"Koch","year":"2022"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1230"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1162\/coli_a_00322"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"ref13","first-page":"74","article-title":"ROUGE: A Package for Automatic Evaluation of Summaries","volume-title":"Text Summarization Branches Out","author":"Lin","year":"2004"},{"key":"ref14","first-page":"249","article-title":"Re-evaluating the role of BLEU in machine translation research","volume-title":"Proc. 11th Conf. Eur. Chapt. Assoc. Comput. Linguistics","author":"Callison-Burch","year":"2006"},{"key":"ref15","first-page":"1","article-title":"An assessment of the accuracy of automatic evaluation in summarization","volume-title":"Proc. Workshop Eval. Metrics System Comparison Autom. Summarization","author":"Owczarzak","year":"2012"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1060"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1230"},{"key":"ref18","first-page":"65","article-title":"METEOR: An automatic metric for MT evaluation with improved correlation with human judgments","volume-title":"Proc. ACL Workshop Intrinsic Extrinsic Eval. Measures Mach. Trans. Summarization","author":"Banerjee","year":"2005"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.213"},{"key":"ref20","article-title":"Bertscore: Evaluating text generation with BERT","volume-title":"Proc. 8th Int. Conf. Learn. Representations","author":"Zhang","year":"2020"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.704"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445922"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00254"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00298"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.findings-acl.98"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1221"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.repl4nlp-1.24"},{"key":"ref28","first-page":"5428","article-title":"BLEURT has universal translations: An analysis of automatic metrics by minimum risk training","volume-title":"Proc. 61st Annu. Meeting Assoc. Comput. Linguistics","author":"Yan","year":"2023"},{"key":"ref29","first-page":"921","article-title":"Learning to evaluate translation beyond english: BLEURT submissions to the WMT metrics 2020 shared task","volume-title":"Proc. 5th Conf. Mach. Transl.","author":"Sellam","year":"2020"},{"key":"ref30","first-page":"3726","article-title":"BERTScore is unfair: On social bias in language model-based metrics for text generation","volume-title":"Proc. Conf. Empirical Methods Natural Lang. Process.","author":"Sun","year":"2022"},{"key":"ref31","first-page":"507","article-title":"A fine-grained analysis of BERTScore","volume-title":"Proc. 6th Conf. Mach. Transl.","author":"Hanna","year":"2021"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/JIOT.2021.3098051"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1162\/coli_a_00356"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00349"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-1419"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.383"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1445"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.wnut-1.3"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.insights-1.1"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.eacl-main.270"},{"key":"ref41","first-page":"88","article-title":"On the systematicity of probing contextualized word representations: The case of hypernymy in BERT","volume-title":"Proc. 9th Joint Conf. Lexical Comput. Semantics","author":"Ravichander","year":"2020"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.573"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1810.04805"},{"key":"ref44","article-title":"Bertscore: Evaluating text generation with BERT","author":"Zhang","year":"2019"},{"key":"ref45","article-title":"TQ-AutoTestan automated test suite for (machine) translation quality","volume-title":"Proc. 11th Int. Conf. Lang. Resour. Eval.","author":"Macketanz","year":"2018"},{"key":"ref46","first-page":"13","article-title":"TSNLP-test suites for natural language processing","author":"Oepen","year":"1997","journal-title":"Linguistic Databases"},{"key":"ref47","first-page":"27","article-title":"PE2rr corpus: Manual error annotation of automatically pre-annotated MT post-edits","volume-title":"Proc. 10th Int. Conf. Lang. Resour. Eval","author":"Popovi","year":"2016"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/W14-1701"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/P15-1068"}],"container-title":["IEEE Open Journal of the Computer Society"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/8782664\/10834807\/10964149.pdf?arnumber=10964149","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,5,27]],"date-time":"2025-05-27T04:34:00Z","timestamp":1748320440000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10964149\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"references-count":49,"URL":"https:\/\/doi.org\/10.1109\/ojcs.2025.3560333","relation":{},"ISSN":["2644-1268"],"issn-type":[{"value":"2644-1268","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025]]}}}