{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,28]],"date-time":"2026-08-28T05:15:58Z","timestamp":1787894158515,"version":"build-2784847793"},"publisher-location":"Cham","reference-count":29,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031446924","type":"print"},{"value":"9783031446931","type":"electronic"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-44693-1_30","type":"book-chapter","created":{"date-parts":[[2023,10,7]],"date-time":"2023-10-07T08:02:39Z","timestamp":1696665759000},"page":"375-386","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":28,"title":["Towards Making the\u00a0Most of\u00a0LLM for\u00a0Translation Quality Estimation"],"prefix":"10.1007","author":[{"given":"Hui","family":"Huang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shuangzhi","family":"Wu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xinnian","family":"Liang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Bing","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yanrui","family":"Shi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Peihao","family":"Wu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Muyun","family":"Yang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tiejun","family":"Zhao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2023,10,8]]},"reference":[{"key":"30_CR1","doi-asserted-by":"crossref","unstructured":"Bang, Y., et al.: A multitask, multilingual, multimodal evaluation of chatgpt on reasoning, hallucination, and interactivity (2023)","DOI":"10.18653\/v1\/2023.ijcnlp-main.45"},{"key":"30_CR2","unstructured":"Barrault, L., et al.: Findings of the 2020 conference on machine translation (WMT20). In: Proceedings of the Fifth Conference on Machine Translation, pp. 1\u201355. Association for Computational Linguistics, November 2020"},{"key":"30_CR3","doi-asserted-by":"crossref","unstructured":"Behnke, H., Fomicheva, M., Specia, L.: Bias mitigation in machine translation quality estimation. In: Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 1475\u20131487. Association for Computational Linguistics, May 2022","DOI":"10.18653\/v1\/2022.acl-long.104"},{"key":"30_CR4","doi-asserted-by":"crossref","unstructured":"Blatz, J., et al.: Confidence estimation for machine translation. In: COLING 2004: Proceedings of the 20th International Conference on Computational Linguistics, pp. 315\u2013321. COLING, Aug 23-Aug 27 2004","DOI":"10.3115\/1220355.1220401"},{"key":"30_CR5","doi-asserted-by":"crossref","unstructured":"Conneau, A., et al.: Unsupervised cross-lingual representation learning at scale. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 8440\u20138451. Association for Computational Linguistics (2020)","DOI":"10.18653\/v1\/2020.acl-main.747"},{"key":"30_CR6","unstructured":"Devlin, J., Chang, M.W., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. In: Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp. 4171\u20134186. Association for Computational Linguistics (Jun 2019)"},{"key":"30_CR7","doi-asserted-by":"publisher","first-page":"539","DOI":"10.1162\/tacl_a_00330","volume":"8","author":"M Fomicheva","year":"2020","unstructured":"Fomicheva, M., et al.: Unsupervised quality estimation for neural machine translation. Trans. Assoc. Comput. Linguistics 8, 539\u2013555 (2020)","journal-title":"Trans. Assoc. Comput. Linguistics"},{"key":"30_CR8","doi-asserted-by":"crossref","unstructured":"Fonseca, E., Yankovskaya, L., Martins, A.F.T., Fishel, M., Federmann, C.: Findings of the WMT 2019 shared tasks on quality estimation. In: Proceedings of the Fourth Conference on Machine Translation (Volume 3: Shared Task Papers, Day 2), pp. 1\u201310. Association for Computational Linguistics, August 2019","DOI":"10.18653\/v1\/W19-5401"},{"key":"30_CR9","unstructured":"Gal, Y., Ghahramani, Z.: Dropout as a bayesian approximation: representing model uncertainty in deep learning. In: International Conference on Machine Learning, pp. 1050\u20131059. PMLR (2016)"},{"key":"30_CR10","unstructured":"Han, L., Jones, G.J., Smeaton, A.F.: Translation quality assessment: a brief survey on manual and automatic methods. arXiv preprint arXiv:2105.03311 (2021)"},{"key":"30_CR11","unstructured":"Kocmi, T., Federmann, C.: Large language models are state-of-the-art evaluators of translation quality (2023)"},{"key":"30_CR12","doi-asserted-by":"crossref","unstructured":"Koco\u0144, J., et al.: Chatgpt: Jack of all trades, master of none (2023)","DOI":"10.2139\/ssrn.4372889"},{"key":"30_CR13","unstructured":"Lee, G., Hou, B., Mandalika, A., Lee, J., Choudhury, S., Srinivasa, S.S.: Bayesian policy optimization for model uncertainty (2019)"},{"key":"30_CR14","unstructured":"Liang, P., et al.: Holistic evaluation of language models (2022)"},{"key":"30_CR15","doi-asserted-by":"publisher","first-page":"0455","DOI":"10.5565\/rev\/tradumatica.77","volume":"12","author":"A Lommel","year":"2014","unstructured":"Lommel, A., Uszkoreit, H., Burchardt, A.: Multidimensional quality metrics (MQM): a framework for declaring and describing translation quality metrics. Tradum\u00e0tica 12, 0455\u20130463 (2014)","journal-title":"Tradum\u00e0tica"},{"key":"30_CR16","doi-asserted-by":"crossref","unstructured":"Lu, Q., Qiu, B., Ding, L., Xie, L., Tao, D.: Error analysis prompting enables human-like translation evaluation in large language models: a case study on chatgpt (2023)","DOI":"10.20944\/preprints202303.0255.v1"},{"key":"30_CR17","doi-asserted-by":"crossref","unstructured":"Min, S., et al.: Rethinking the role of demonstrations: what makes in-context learning work? In: Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing, pp. 11048\u201311064. Association for Computational Linguistics, December 2022","DOI":"10.18653\/v1\/2022.emnlp-main.759"},{"key":"30_CR18","doi-asserted-by":"publisher","unstructured":"Papineni, K., Roukos, S., Ward, T., Zhu, W.J.: Bleu: a method for automatic evaluation of machine translation. In: Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics, pp. 311\u2013318. Association for Computational Linguistics, July 2002. https:\/\/doi.org\/10.3115\/1073083.1073135","DOI":"10.3115\/1073083.1073135"},{"key":"30_CR19","doi-asserted-by":"crossref","unstructured":"Peng, K., et al.: Towards making the most of chatgpt for machine translation. arXiv preprint arXiv:2303.13780 (2023)","DOI":"10.2139\/ssrn.4390455"},{"key":"30_CR20","doi-asserted-by":"crossref","unstructured":"Qin, C., Zhang, A., Zhang, Z., Chen, J., Yasunaga, M., Yang, D.: Is chatgpt a general-purpose natural language processing task solver? (2023)","DOI":"10.18653\/v1\/2023.emnlp-main.85"},{"key":"30_CR21","doi-asserted-by":"crossref","unstructured":"Ranasinghe, T., Orasan, C., Mitkov, R.: TransQuest: translation quality estimation with cross-lingual transformers. In: Proceedings of the 28th International Conference on Computational Linguistics, pp. 5070\u20135081. International Committee on Computational Linguistics (Dec 2020)","DOI":"10.18653\/v1\/2020.coling-main.445"},{"key":"30_CR22","doi-asserted-by":"crossref","unstructured":"Rei, R., Stewart, C., Farinha, A.C., Lavie, A.: COMET: a neural framework for MT evaluation. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 2685\u20132702. Association for Computational Linguistics, November 2020","DOI":"10.18653\/v1\/2020.emnlp-main.213"},{"key":"30_CR23","doi-asserted-by":"crossref","unstructured":"Robertson, S., Zaragoza, H., et al.: The probabilistic relevance framework: Bm25 and beyond. Found. Trends Inf. Retrieval 3(4), 333\u2013389 (2009)","DOI":"10.1561\/1500000019"},{"key":"30_CR24","doi-asserted-by":"crossref","unstructured":"Sun, S., Guzm\u00e1n, F., Specia, L.: Are we estimating or guesstimating translation quality? In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 6262\u20136267. Association for Computational Linguistics, July 2020","DOI":"10.18653\/v1\/2020.acl-main.558"},{"key":"30_CR25","doi-asserted-by":"crossref","unstructured":"Wang, K., Shi, Y., Wang, J., Zhang, Y., Zhao, Y., Zheng, X.: Beyond glass-box features: uncertainty quantification enhanced quality estimation for neural machine translation. In: Findings of the Association for Computational Linguistics: EMNLP 2021, pp. 4687\u20134698. Association for Computational Linguistics, Novenber 2021","DOI":"10.18653\/v1\/2021.findings-emnlp.401"},{"key":"30_CR26","doi-asserted-by":"crossref","unstructured":"Wang, S., Liu, Y., Wang, C., Luan, H., Sun, M.: Improving back-translation with uncertainty-based confidence estimation. In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), pp. 791\u2013802. Association for Computational Linguistics (Nov 2019)","DOI":"10.18653\/v1\/D19-1073"},{"key":"30_CR27","doi-asserted-by":"crossref","unstructured":"Xiao, Y., Wang, W.Y.: Quantifying uncertainties in natural language processing tasks. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol. 33, pp. 7322\u20137329 (2019)","DOI":"10.1609\/aaai.v33i01.33017322"},{"key":"30_CR28","unstructured":"Zerva, C., et al.: Findings of the WMT 2022 shared task on quality estimation. In: Proceedings of the Seventh Conference on Machine Translation (WMT), pp. 69\u201399. Association for Computational Linguistics, December 2022"},{"key":"30_CR29","unstructured":"Zhang, B., Haddow, B., Birch, A.: Prompting large language model for machine translation: a case study. arXiv preprint arXiv:2301.07069 (2023)"}],"container-title":["Lecture Notes in Computer Science","Natural Language Processing and Chinese Computing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-44693-1_30","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,30]],"date-time":"2024-10-30T08:54:35Z","timestamp":1730278475000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-44693-1_30"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031446924","9783031446931"],"references-count":29,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-44693-1_30","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"8 October 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"NLPCC","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"CCF International Conference on Natural Language Processing and Chinese Computing","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Foshan","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"China","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"12 October 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"15 October 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"12","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"nlpcc2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/tcci.ccf.org.cn\/conference\/2023\/index.php","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Softconf","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"478","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"143","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"30% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}