{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,25]],"date-time":"2025-03-25T14:33:30Z","timestamp":1742913210007,"version":"3.40.3"},"publisher-location":"Cham","reference-count":31,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031723438"},{"type":"electronic","value":"9783031723445"}],"license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024]]},"DOI":"10.1007\/978-3-031-72344-5_25","type":"book-chapter","created":{"date-parts":[[2024,9,16]],"date-time":"2024-09-16T14:03:01Z","timestamp":1726495381000},"page":"371-385","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["REM: A Ranking-Based Automatic Evaluation Method for\u00a0LLMs"],"prefix":"10.1007","author":[{"given":"Jintao","family":"Yang","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yushan","family":"Tan","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Wenpeng","family":"Hu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zonghao","family":"Yang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xian","family":"Zhou","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhunchen","family":"Luo","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Wei","family":"Luo","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,9,17]]},"reference":[{"key":"25_CR1","unstructured":"Achiam, J., et\u00a0al.: Gpt-4 technical report. arXiv preprint arXiv:2303.08774 (2023)"},{"key":"25_CR2","unstructured":"Anil, R., et\u00a0al.: Palm 2 technical report. arXiv preprint arXiv:2305.10403 (2023)"},{"key":"25_CR3","doi-asserted-by":"crossref","unstructured":"Bang, Y., et\u00a0al.: A multitask, multilingual, multimodal evaluation of chatgpt on reasoning, hallucination, and interactivity. arXiv preprint arXiv:2302.04023 (2023)","DOI":"10.18653\/v1\/2023.ijcnlp-main.45"},{"key":"25_CR4","first-page":"1877","volume":"33","author":"T Brown","year":"2020","unstructured":"Brown, T., et al.: Language models are few-shot learners. Adv. Neural. Inf. Process. Syst. 33, 1877\u20131901 (2020)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"25_CR5","unstructured":"Bubeck, S., et\u00a0al.: Sparks of artificial general intelligence: early experiments with gpt-4. arXiv preprint arXiv:2303.12712 (2023)"},{"key":"25_CR6","unstructured":"Chang, Y., et\u00a0al.: A survey on evaluation of large language models. arXiv preprint arXiv:2307.03109 (2023)"},{"key":"25_CR7","unstructured":"Chiang, W.L., et\u00a0al.: Chatbot arena: an open platform for evaluating llms by human preference. arXiv preprint arXiv:2403.04132 (2024)"},{"key":"25_CR8","unstructured":"Chowdhery, A., et\u00a0al.: Palm: scaling language modeling with pathways. arXiv preprint arXiv:2204.02311 (2022)"},{"issue":"8","key":"25_CR9","first-page":"242","volume":"22","author":"AE Elo","year":"1967","unstructured":"Elo, A.E.: The proposed uscf rating system, its development, theory, and applications. Chess Life 22(8), 242\u2013247 (1967)","journal-title":"Chess Life"},{"issue":"6","key":"25_CR10","doi-asserted-by":"publisher","first-page":"673","DOI":"10.1080\/02664760120059219","volume":"28","author":"ME Glickman","year":"2001","unstructured":"Glickman, M.E.: Dynamic paired comparison models with stochastic variances. J. Appl. Stat. 28(6), 673\u2013689 (2001)","journal-title":"J. Appl. Stat."},{"key":"25_CR11","unstructured":"Hendrycks, D., et\u00a0al.: Measuring coding challenge competence with apps. arXiv preprint arXiv:2105.09938 (2021)"},{"key":"25_CR12","unstructured":"Hendrycks, D., et al.: Measuring massive multitask language understanding. arXiv preprint arXiv:2009.03300 (2020)"},{"key":"25_CR13","unstructured":"Hendrycks, D., Burns, C., Chen, A., Ball, S.: Cuad: an expert-annotated NLP dataset for legal contract review. arXiv preprint arXiv:2103.06268 (2021)"},{"key":"25_CR14","unstructured":"Huang, Y., et\u00a0al.: C-eval: a multi-level multi-discipline Chinese evaluation suite for foundation models. arXiv preprint arXiv:2305.08322 (2023)"},{"key":"25_CR15","unstructured":"Li, X., et al.: Alpacaeval: an automatic evaluator of instruction-following models (2023)"},{"key":"25_CR16","unstructured":"Liang, P., et\u00a0al.: Holistic evaluation of language models. arXiv preprint arXiv:2211.09110 (2022)"},{"key":"25_CR17","doi-asserted-by":"crossref","unstructured":"Lin, Y.T., Chen, Y.N.: LLM-EVAL: unified multi-dimensional automatic evaluation for open-domain conversations with large language models. arXiv preprint arXiv:2305.13711 (2023)","DOI":"10.18653\/v1\/2023.nlp4convai-1.5"},{"key":"25_CR18","doi-asserted-by":"crossref","unstructured":"Qin, C., Zhang, A., Zhang, Z., Chen, J., Yasunaga, M., Yang, D.: Is chatgpt a general-purpose natural language processing task solver? arXiv preprint arXiv:2302.06476 (2023)","DOI":"10.18653\/v1\/2023.emnlp-main.85"},{"key":"25_CR19","unstructured":"Singhal, K., et\u00a0al.: Large language models encode clinical knowledge. arXiv preprint arXiv:2212.13138 (2022)"},{"key":"25_CR20","unstructured":"Srivastava, A., et\u00a0al.: Beyond the imitation game: quantifying and extrapolating the capabilities of language models. arXiv preprint arXiv:2206.04615 (2022)"},{"key":"25_CR21","unstructured":"Tian, J., et\u00a0al.: Chatplug: open-domain generative dialogue system with internet-augmented instruction tuning for digital human. arXiv preprint arXiv:2304.07849 (2023)"},{"key":"25_CR22","unstructured":"Touvron, H., et\u00a0al.: Llama 2: open foundation and fine-tuned chat models. arXiv preprint arXiv:2307.09288 (2023)"},{"key":"25_CR23","doi-asserted-by":"crossref","unstructured":"Wang, L., Lyu, C., Ji, T., Zhang, Z., Yu, D., Shi, S., Tu, Z.: Document-level machine translation with large language models. arXiv preprint arXiv:2304.02210 (2023)","DOI":"10.18653\/v1\/2023.emnlp-main.1036"},{"key":"25_CR24","unstructured":"Wang, P., et al.: Large language models are not fair evaluators. arXiv preprint arXiv:2305.17926 (2023)"},{"key":"25_CR25","unstructured":"Wang, Y., et\u00a0al.: Pandalm: an automatic evaluation benchmark for llm instruction tuning optimization. arXiv preprint arXiv:2306.05087 (2023)"},{"key":"25_CR26","unstructured":"Xu, L., et al.: Superclue: a comprehensive Chinese large language model benchmark. arXiv preprint arXiv:2307.15020 (2023)"},{"key":"25_CR27","unstructured":"Yang, K.C., Menczer, F.: Large language models can rate news outlet credibility. arXiv preprint arXiv:2304.00228 (2023)"},{"key":"25_CR28","unstructured":"Yu, J., et\u00a0al.: Kola: carefully benchmarking world knowledge of large language models. arXiv preprint arXiv:2306.09296 (2023)"},{"key":"25_CR29","unstructured":"Zeng, A., et\u00a0al.: Glm-130b: an open bilingual pre-trained model. arXiv preprint arXiv:2210.02414 (2022)"},{"key":"25_CR30","unstructured":"Zhang, X., Li, C., Zong, Y., Ying, Z., He, L., Qiu, X.: Evaluating the performance of large language models on gaokao benchmark. arXiv preprint arXiv:2305.12474 (2023)"},{"key":"25_CR31","unstructured":"Zheng, L., et\u00a0al.: Judging llm-as-a-judge with mt-bench and chatbot arena. arXiv preprint arXiv:2306.05685 (2023)"}],"container-title":["Lecture Notes in Computer Science","Artificial Neural Networks and Machine Learning \u2013 ICANN 2024"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-72344-5_25","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,16]],"date-time":"2024-09-16T14:08:34Z","timestamp":1726495714000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-72344-5_25"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"ISBN":["9783031723438","9783031723445"],"references-count":31,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-72344-5_25","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2024]]},"assertion":[{"value":"17 September 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ICANN","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Artificial Neural Networks","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Lugano","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Switzerland","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"17 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"20 September 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"33","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"icann2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}