{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,19]],"date-time":"2026-06-19T23:54:13Z","timestamp":1781913253646,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":75,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,5,11]],"date-time":"2024-05-11T00:00:00Z","timestamp":1715385600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-sa\/4.0\/"}],"funder":[{"DOI":"10.13039\/https:\/\/doi.org\/10.13039\/501100014188","name":"Ministry of Science and ICT, South Korea","doi-asserted-by":"publisher","award":["2021-0-01347"],"award-info":[{"award-number":["2021-0-01347"]}],"id":[{"id":"10.13039\/https:\/\/doi.org\/10.13039\/501100014188","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,5,11]]},"DOI":"10.1145\/3613904.3642216","type":"proceedings-article","created":{"date-parts":[[2024,5,11]],"date-time":"2024-05-11T08:38:25Z","timestamp":1715416705000},"page":"1-21","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":82,"title":["EvalLM: Interactive Evaluation of Large Language Model Prompts on User-Defined Criteria"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9078-6032","authenticated-orcid":false,"given":"Tae Soo","family":"Kim","sequence":"first","affiliation":[{"name":"School of Computing, KAIST, Korea, Republic of"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7491-986X","authenticated-orcid":false,"given":"Yoonjoo","family":"Lee","sequence":"additional","affiliation":[{"name":"School of Computing, KAIST, Korea, Republic of"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4604-7299","authenticated-orcid":false,"given":"Jamin","family":"Shin","sequence":"additional","affiliation":[{"name":"NAVER AI Lab, Korea, Republic of"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2681-2774","authenticated-orcid":false,"given":"Young-Ho","family":"Kim","sequence":"additional","affiliation":[{"name":"NAVER AI Lab, Korea, Republic of"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6348-4127","authenticated-orcid":false,"given":"Juho","family":"Kim","sequence":"additional","affiliation":[{"name":"School of Computing, KAIST, Korea, Republic of"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,5,11]]},"reference":[{"key":"e_1_3_3_3_1_1","volume-title":"RL4F: Generating Natural Language Feedback with Reinforcement Learning for Repairing Model Outputs. arXiv preprint arXiv:2305.08844","author":"Aky\u00fcrek Afra\u00a0Feyza","year":"2023","unstructured":"Afra\u00a0Feyza Aky\u00fcrek, Ekin Aky\u00fcrek, Aman Madaan, Ashwin Kalyan, Peter Clark, Derry Wijaya, and Niket Tandon. 2023. RL4F: Generating Natural Language Feedback with Reinforcement Learning for Repairing Model Outputs. arXiv preprint arXiv:2305.08844 (2023)."},{"key":"e_1_3_3_3_3_1","doi-asserted-by":"publisher","DOI":"10.1017\/pan.2023.2"},{"key":"e_1_3_3_3_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3589955"},{"key":"e_1_3_3_3_5_1","volume-title":"Constitutional ai: Harmlessness from ai feedback. arXiv preprint arXiv:2212.08073","author":"Bai Yuntao","year":"2022","unstructured":"Yuntao Bai, Saurav Kadavath, Sandipan Kundu, Amanda Askell, Jackson Kernion, Andy Jones, Anna Chen, Anna Goldie, Azalia Mirhoseini, Cameron McKinnon, 2022. Constitutional ai: Harmlessness from ai feedback. arXiv preprint arXiv:2212.08073 (2022)."},{"key":"e_1_3_3_3_6_1","volume-title":"Benchmarking Foundation Models with Language-Model-as-an-Examiner. arXiv preprint arXiv:2306.04181","author":"Bai Yushi","year":"2023","unstructured":"Yushi Bai, Jiahao Ying, Yixin Cao, Xin Lv, Yuze He, Xiaozhi Wang, Jifan Yu, Kaisheng Zeng, Yijia Xiao, Haozhe Lyu, 2023. Benchmarking Foundation Models with Language-Model-as-an-Examiner. arXiv preprint arXiv:2306.04181 (2023)."},{"key":"e_1_3_3_3_7_1","doi-asserted-by":"publisher","DOI":"10.3389\/fpubh.2018.00149"},{"key":"e_1_3_3_3_8_1","volume-title":"On the opportunities and risks of foundation models. arXiv preprint arXiv:2108.07258","author":"Bommasani Rishi","year":"2021","unstructured":"Rishi Bommasani, Drew\u00a0A Hudson, Ehsan Adeli, Russ Altman, Simran Arora, Sydney von Arx, Michael\u00a0S Bernstein, Jeannette Bohg, Antoine Bosselut, Emma Brunskill, 2021. On the opportunities and risks of foundation models. arXiv preprint arXiv:2108.07258 (2021)."},{"key":"e_1_3_3_3_9_1","unstructured":"Tom\u00a0B. Brown Benjamin Mann Nick Ryder Melanie Subbiah Jared Kaplan Prafulla Dhariwal Arvind Neelakantan Pranav Shyam Girish Sastry Amanda Askell Sandhini Agarwal Ariel Herbert-Voss Gretchen Krueger Tom Henighan Rewon Child Aditya Ramesh Daniel\u00a0M. Ziegler Jeffrey Wu Clemens Winter Christopher Hesse Mark Chen Eric Sigler Mateusz Litwin Scott Gray Benjamin Chess Jack Clark Christopher Berner Sam McCandlish Alec Radford Ilya Sutskever and Dario Amodei. 2020. Language Models are Few-Shot Learners. arxiv:2005.14165\u00a0[cs.CL]"},{"key":"e_1_3_3_3_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3479569"},{"key":"e_1_3_3_3_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581268"},{"key":"e_1_3_3_3_12_1","doi-asserted-by":"crossref","unstructured":"Cheng-Han Chiang and Hung yi Lee. 2023. Can Large Language Models Be an Alternative to Human Evaluations?arxiv:2305.01937\u00a0[cs.CL]","DOI":"10.18653\/v1\/2023.acl-long.870"},{"key":"e_1_3_3_3_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501819"},{"key":"e_1_3_3_3_14_1","doi-asserted-by":"publisher","unstructured":"Elizabeth Clark Tal August Sofia Serrano Nikita Haduong Suchin Gururangan and Noah\u00a0A. Smith. 2021. All That\u2019s \u2018Human\u2019 Is Not Gold: Evaluating Human Evaluation of Generated Text. In Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers). Association for Computational Linguistics Online 7282\u20137296. https:\/\/doi.org\/10.18653\/v1\/2021.acl-long.565","DOI":"10.18653\/v1"},{"key":"e_1_3_3_3_15_1","unstructured":"Jiaxi Cui Zongjian Li Yang Yan Bohua Chen and Li Yuan. 2023. ChatLaw: Open-Source Legal Large Language Model with Integrated External Knowledge Bases. arxiv:2306.16092\u00a0[cs.CL]"},{"key":"e_1_3_3_3_16_1","doi-asserted-by":"crossref","unstructured":"Shizhe Diao Rui Pan Hanze Dong Ka\u00a0Shun Shum Jipeng Zhang Wei Xiong and Tong Zhang. 2023. LMFlow: An Extensible Toolkit for Finetuning and Inference of Large Foundation Models. arxiv:2306.12420\u00a0[cs.CL]","DOI":"10.18653\/v1\/2024.naacl-demo.12"},{"key":"e_1_3_3_3_17_1","volume-title":"Is GPT-3 text indistinguishable from human text? SCARECROW: A framework for scrutinizing machine text. arXiv preprint arXiv:2107.01294","author":"Dou Yao","year":"2021","unstructured":"Yao Dou, Maxwell Forbes, Rik Koncel-Kedziorski, Noah\u00a0A Smith, and Yejin Choi. 2021. Is GPT-3 text indistinguishable from human text? SCARECROW: A framework for scrutinizing machine text. arXiv preprint arXiv:2107.01294 (2021)."},{"key":"e_1_3_3_3_18_1","unstructured":"Yann Dubois Xuechen Li Rohan Taori Tianyi Zhang Ishaan Gulrajani Jimmy Ba Carlos Guestrin Percy Liang and Tatsunori\u00a0B. Hashimoto. 2023. AlpacaFarm: A Simulation Framework for Methods that Learn from Human Feedback. arxiv:2305.14387\u00a0[cs.LG]"},{"key":"e_1_3_3_3_19_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00373"},{"key":"e_1_3_3_3_20_1","volume-title":"Hierarchical neural story generation. arXiv preprint arXiv:1805.04833","author":"Fan Angela","year":"2018","unstructured":"Angela Fan, Mike Lewis, and Yann Dauphin. 2018. Hierarchical neural story generation. arXiv preprint arXiv:1805.04833 (2018)."},{"key":"e_1_3_3_3_21_1","volume-title":"\u00a0T. Martins","author":"Fernandes Patrick","year":"2023","unstructured":"Patrick Fernandes, Aman Madaan, Emmy Liu, Ant\u00f3nio Farinhas, Pedro\u00a0Henrique Martins, Amanda Bertsch, Jos\u00e9 G.\u00a0C. de Souza, Shuyan Zhou, Tongshuang Wu, Graham Neubig, and Andr\u00e9 F.\u00a0T. Martins. 2023. Bridging the Gap: A Survey on Integrating (Human) Feedback for Natural Language Generation. arxiv:2305.00955\u00a0[cs.CL]"},{"key":"e_1_3_3_3_22_1","volume-title":"Gptscore: Evaluate as you desire. arXiv preprint arXiv:2302.04166","author":"Fu Jinlan","year":"2023","unstructured":"Jinlan Fu, See-Kiong Ng, Zhengbao Jiang, and Pengfei Liu. 2023. Gptscore: Evaluate as you desire. arXiv preprint arXiv:2302.04166 (2023)."},{"key":"e_1_3_3_3_23_1","doi-asserted-by":"publisher","DOI":"10.1613\/jair.1.13715"},{"key":"e_1_3_3_3_24_1","unstructured":"Karthik Gopalakrishnan Behnam Hedayatnia Qinlang Chen Anna Gottardi Sanjeev Kwatra Anu Venkatesh Raefer Gabriel and Dilek Hakkani-Tur. 2023. Topical-Chat: Towards Knowledge-Grounded Open-Domain Conversations. arxiv:2308.11995\u00a0[cs.CL]"},{"key":"e_1_3_3_3_25_1","doi-asserted-by":"publisher","unstructured":"Sandra\u00a0G. Hart and Lowell\u00a0E. Staveland. 1988. Development of NASA-TLX (Task Load Index): Results of Empirical and Theoretical Research. In Human Mental Workload Peter\u00a0A. Hancock and Najmedin Meshkati (Eds.). Advances in Psychology Vol.\u00a052. North-Holland 139\u2013183. https:\/\/doi.org\/10.1016\/S0166-4115(08)62386-9","DOI":"10.1016\/S0166-4115(08)62386-9"},{"key":"e_1_3_3_3_26_1","unstructured":"Edward\u00a0J. Hu Yelong Shen Phillip Wallis Zeyuan Allen-Zhu Yuanzhi Li Shean Wang Lu Wang and Weizhu Chen. 2021. LoRA: Low-Rank Adaptation of Large Language Models. arxiv:2106.09685\u00a0[cs.CL]"},{"key":"e_1_3_3_3_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3503564"},{"key":"e_1_3_3_3_28_1","volume-title":"GENIE: Toward reproducible and standardized human evaluation for text generation. arXiv preprint arXiv:2101.06561","author":"Khashabi Daniel","year":"2021","unstructured":"Daniel Khashabi, Gabriel Stanovsky, Jonathan Bragg, Nicholas Lourie, Jungo Kasai, Yejin Choi, Noah\u00a0A Smith, and Daniel\u00a0S Weld. 2021. GENIE: Toward reproducible and standardized human evaluation for text generation. arXiv preprint arXiv:2101.06561 (2021)."},{"key":"e_1_3_3_3_29_1","unstructured":"Sungdong Kim Sanghwan Bae Jamin Shin Soyoung Kang Donghyun Kwak Kang\u00a0Min Yoo and Minjoon Seo. 2023. Aligning Large Language Models through Synthetic Feedback. arxiv:2305.13735\u00a0[cs.CL]"},{"key":"e_1_3_3_3_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606833"},{"key":"e_1_3_3_3_31_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2018340118"},{"key":"e_1_3_3_3_32_1","volume-title":"LongEval: Guidelines for human evaluation of faithfulness in long-form summarization. arXiv preprint arXiv:2301.13298","author":"Krishna Kalpesh","year":"2023","unstructured":"Kalpesh Krishna, Erin Bransom, Bailey Kuehl, Mohit Iyyer, Pradeep Dasigi, Arman Cohan, and Kyle Lo. 2023. LongEval: Guidelines for human evaluation of faithfulness in long-form summarization. arXiv preprint arXiv:2301.13298 (2023)."},{"key":"e_1_3_3_3_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581369"},{"key":"e_1_3_3_3_34_1","volume-title":"ACUTE-EVAL: Improved Dialogue Evaluation with Optimized Questions and Multi-Turn Comparisons. arXiv preprint arXiv:1909.03087","author":"Li Margaret","year":"2019","unstructured":"Margaret Li, Jason Weston, and Stephen Roller. 2019. ACUTE-EVAL: Improved Dialogue Evaluation with Optimized Questions and Multi-Turn Comparisons. arXiv preprint arXiv:1909.03087 (2019)."},{"key":"e_1_3_3_3_35_1","volume-title":"PRD: Peer Rank and Discussion Improve Large Language Model based Evaluations. arXiv preprint arXiv:2307.02762","author":"Li Ruosen","year":"2023","unstructured":"Ruosen Li, Teerth Patel, and Xinya Du. 2023. PRD: Peer Rank and Discussion Improve Large Language Model based Evaluations. arXiv preprint arXiv:2307.02762 (2023)."},{"key":"e_1_3_3_3_36_1","volume-title":"ROUGE: A Package for Automatic Evaluation of Summaries. In Text Summarization Branches Out","author":"Lin Chin-Yew","year":"2004","unstructured":"Chin-Yew Lin. 2004. ROUGE: A Package for Automatic Evaluation of Summaries. In Text Summarization Branches Out. Association for Computational Linguistics, Barcelona, Spain, 74\u201381. https:\/\/aclanthology.org\/W04-1013"},{"key":"e_1_3_3_3_37_1","volume-title":"What Makes Good In-Context Examples for GPT-3 ?arXiv preprint arXiv:2101.06804","author":"Liu Jiachang","year":"2021","unstructured":"Jiachang Liu, Dinghan Shen, Yizhe Zhang, Bill Dolan, Lawrence Carin, and Weizhu Chen. 2021. What Makes Good In-Context Examples for GPT-3 ?arXiv preprint arXiv:2101.06804 (2021)."},{"key":"e_1_3_3_3_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580817"},{"key":"e_1_3_3_3_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3560815"},{"key":"e_1_3_3_3_40_1","doi-asserted-by":"crossref","unstructured":"Yang Liu Dan Iter Yichong Xu Shuohang Wang Ruochen Xu and Chenguang Zhu. 2023. G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment. arxiv:2303.16634\u00a0[cs.CL]","DOI":"10.18653\/v1\/2023.emnlp-main.153"},{"key":"e_1_3_3_3_41_1","volume-title":"Fantastically ordered prompts and where to find them: Overcoming few-shot prompt order sensitivity. arXiv preprint arXiv:2104.08786","author":"Lu Yao","year":"2021","unstructured":"Yao Lu, Max Bartolo, Alastair Moore, Sebastian Riedel, and Pontus Stenetorp. 2021. Fantastically ordered prompts and where to find them: Overcoming few-shot prompt order sensitivity. arXiv preprint arXiv:2104.08786 (2021)."},{"key":"e_1_3_3_3_42_1","volume-title":"Self-refine: Iterative refinement with self-feedback. arXiv preprint arXiv:2303.17651","author":"Madaan Aman","year":"2023","unstructured":"Aman Madaan, Niket Tandon, Prakhar Gupta, Skyler Hallinan, Luyu Gao, Sarah Wiegreffe, Uri Alon, Nouha Dziri, Shrimai Prabhumoye, Yiming Yang, 2023. Self-refine: Iterative refinement with self-feedback. arXiv preprint arXiv:2303.17651 (2023)."},{"key":"e_1_3_3_3_43_1","volume-title":"PEFT: State-of-the-art Parameter-Efficient Fine-Tuning methods. https:\/\/github.com\/huggingface\/peft.","author":"Mangrulkar Sourab","year":"2022","unstructured":"Sourab Mangrulkar, Sylvain Gugger, Lysandre Debut, Younes Belkada, and Sayak Paul. 2022. PEFT: State-of-the-art Parameter-Efficient Fine-Tuning methods. https:\/\/github.com\/huggingface\/peft."},{"key":"e_1_3_3_3_44_1","volume-title":"Perturbation, Testing and Iteration using Visual Analytics for Large Language Models. arXiv preprint arXiv:2304.01964","author":"Mishra Aditi","year":"2023","unstructured":"Aditi Mishra, Utkarsh Soni, Anjana Arunkumar, Jinbin Huang, Bum\u00a0Chul Kwon, and Chris Bryan. 2023. PromptAid: Prompt Exploration, Perturbation, Testing and Iteration using Visual Analytics for Large Language Models. arXiv preprint arXiv:2304.01964 (2023)."},{"key":"e_1_3_3_3_45_1","volume-title":"Abstractive text summarization using sequence-to-sequence rnns and beyond. arXiv preprint arXiv:1602.06023","author":"Nallapati Ramesh","year":"2016","unstructured":"Ramesh Nallapati, Bowen Zhou, Caglar Gulcehre, Bing Xiang, 2016. Abstractive text summarization using sequence-to-sequence rnns and beyond. arXiv preprint arXiv:1602.06023 (2016)."},{"key":"e_1_3_3_3_47_1","unstructured":"Long Ouyang Jeff Wu Xu Jiang Diogo Almeida Carroll\u00a0L. Wainwright Pamela Mishkin Chong Zhang Sandhini Agarwal Katarina Slama Alex Ray John Schulman Jacob Hilton Fraser Kelton Luke Miller Maddie Simens Amanda Askell Peter Welinder Paul Christiano Jan Leike and Ryan Lowe. 2022. Training language models to follow instructions with human feedback. arxiv:2203.02155\u00a0[cs.CL]"},{"key":"e_1_3_3_3_48_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"e_1_3_3_3_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3526113.3545616"},{"key":"e_1_3_3_3_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580907"},{"key":"e_1_3_3_3_51_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.230"},{"key":"e_1_3_3_3_52_1","volume-title":"Beyond accuracy: Behavioral testing of NLP models with CheckList. arXiv preprint arXiv:2005.04118","author":"Ribeiro Marco\u00a0Tulio","year":"2020","unstructured":"Marco\u00a0Tulio Ribeiro, Tongshuang Wu, Carlos Guestrin, and Sameer Singh. 2020. Beyond accuracy: Behavioral testing of NLP models with CheckList. arXiv preprint arXiv:2005.04118 (2020)."},{"key":"e_1_3_3_3_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580790"},{"key":"e_1_3_3_3_54_1","doi-asserted-by":"publisher","DOI":"10.1002\/hrm.21852"},{"key":"e_1_3_3_3_55_1","volume-title":"Learning to retrieve prompts for in-context learning. arXiv preprint arXiv:2112.08633","author":"Rubin Ohad","year":"2021","unstructured":"Ohad Rubin, Jonathan Herzig, and Jonathan Berant. 2021. Learning to retrieve prompts for in-context learning. arXiv preprint arXiv:2112.08633 (2021)."},{"key":"e_1_3_3_3_56_1","volume-title":"Whose opinions do language models reflect?arXiv preprint arXiv:2303.17548","author":"Santurkar Shibani","year":"2023","unstructured":"Shibani Santurkar, Esin Durmus, Faisal Ladhak, Cinoo Lee, Percy Liang, and Tatsunori Hashimoto. 2023. Whose opinions do language models reflect?arXiv preprint arXiv:2303.17548 (2023)."},{"key":"e_1_3_3_3_57_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2022.3209479"},{"key":"e_1_3_3_3_58_1","unstructured":"Zhiqing Sun Yikang Shen Qinhong Zhou Hongxin Zhang Zhenfang Chen David Cox Yiming Yang and Chuang Gan. 2023. Principle-Driven Self-Alignment of Language Models from Scratch with Minimal Human Supervision. arxiv:2305.03047\u00a0[cs.LG]"},{"key":"e_1_3_3_3_59_1","unstructured":"Hugo Touvron Thibaut Lavril Gautier Izacard Xavier Martinet Marie-Anne Lachaux Timoth\u00e9e Lacroix Baptiste Rozi\u00e8re Naman Goyal Eric Hambro Faisal Azhar Aurelien Rodriguez Armand Joulin Edouard Grave and Guillaume Lample. 2023. LLaMA: Open and Efficient Foundation Language Models. arxiv:2302.13971\u00a0[cs.CL]"},{"key":"e_1_3_3_3_60_1","unstructured":"Hugo Touvron Louis Martin Kevin Stone Peter Albert Amjad Almahairi Yasmine Babaei Nikolay Bashlykov Soumya Batra Prajjwal Bhargava Shruti Bhosale Dan Bikel Lukas Blecher Cristian\u00a0Canton Ferrer Moya Chen Guillem Cucurull David Esiobu Jude Fernandes Jeremy Fu Wenyin Fu Brian Fuller Cynthia Gao Vedanuj Goswami Naman Goyal Anthony Hartshorn Saghar Hosseini Rui Hou Hakan Inan Marcin Kardas Viktor Kerkez Madian Khabsa Isabel Kloumann Artem Korenev Punit\u00a0Singh Koura Marie-Anne Lachaux Thibaut Lavril Jenya Lee Diana Liskovich Yinghai Lu Yuning Mao Xavier Martinet Todor Mihaylov Pushkar Mishra Igor Molybog Yixin Nie Andrew Poulton Jeremy Reizenstein Rashi Rungta Kalyan Saladi Alan Schelten Ruan Silva Eric\u00a0Michael Smith Ranjan Subramanian Xiaoqing\u00a0Ellen Tan Binh Tang Ross Taylor Adina Williams Jian\u00a0Xiang Kuan Puxin Xu Zheng Yan Iliyan Zarov Yuchen Zhang Angela Fan Melanie Kambadur Sharan Narang Aurelien Rodriguez Robert Stojnic Sergey Edunov and Thomas Scialom. 2023. Llama 2: Open Foundation and Fine-Tuned Chat Models. arxiv:2307.09288\u00a0[cs.CL]"},{"key":"e_1_3_3_3_61_1","volume-title":"Large language models are not fair evaluators. arXiv preprint arXiv:2305.17926","author":"Wang Peiyi","year":"2023","unstructured":"Peiyi Wang, Lei Li, Liang Chen, Dawei Zhu, Binghuai Lin, Yunbo Cao, Qi Liu, Tianyu Liu, and Zhifang Sui. 2023. Large language models are not fair evaluators. arXiv preprint arXiv:2305.17926 (2023)."},{"key":"e_1_3_3_3_62_1","volume-title":"Shepherd: A Critic for Language Model Generation. arxiv:2308.04592\u00a0[cs.CL]","author":"Wang Tianlu","year":"2023","unstructured":"Tianlu Wang, Ping Yu, Xiaoqing\u00a0Ellen Tan, Sean O\u2019Brien, Ramakanth Pasunuru, Jane Dwivedi-Yu, Olga Golovneva, Luke Zettlemoyer, Maryam Fazel-Zarandi, and Asli Celikyilmaz. 2023. Shepherd: A Critic for Language Model Generation. arxiv:2308.04592\u00a0[cs.CL]"},{"key":"e_1_3_3_3_63_1","unstructured":"Jing Wei Sungdong Kim Hyunhoon Jung and Young-Ho Kim. 2023. Leveraging Large Language Models to Power Chatbots for Collecting User Self-Reported Data. arxiv:2301.05843\u00a0[cs.HC]"},{"key":"e_1_3_3_3_64_1","first-page":"24824","article-title":"Chain-of-thought prompting elicits reasoning in large language models","volume":"35","author":"Wei Jason","year":"2022","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Fei Xia, Ed Chi, Quoc\u00a0V Le, Denny Zhou, 2022. Chain-of-thought prompting elicits reasoning in large language models. Advances in Neural Information Processing Systems 35 (2022), 24824\u201324837.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_3_3_65_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2019.2934619"},{"key":"e_1_3_3_3_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/3581641.3584059"},{"key":"e_1_3_3_3_67_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_3_3_68_1","volume-title":"Polyjuice: Generating counterfactuals for explaining, evaluating, and improving models. arXiv preprint arXiv:2101.00288","author":"Wu Tongshuang","year":"2021","unstructured":"Tongshuang Wu, Marco\u00a0Tulio Ribeiro, Jeffrey Heer, and Daniel\u00a0S Weld. 2021. Polyjuice: Generating counterfactuals for explaining, evaluating, and improving models. arXiv preprint arXiv:2101.00288 (2021)."},{"key":"e_1_3_3_3_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517582"},{"key":"e_1_3_3_3_70_1","unstructured":"Ziang Xiao Susu Zhang Vivian Lai and Q.\u00a0Vera Liao. 2023. Evaluating NLG Evaluation Metrics: A Measurement Theory Perspective. arxiv:2305.14889\u00a0[cs.CL]"},{"key":"e_1_3_3_3_71_1","volume-title":"FLASK: Fine-grained Language Model Evaluation based on Alignment Skill Sets. arXiv preprint arXiv:2307.10928","author":"Ye Seonghyeon","year":"2023","unstructured":"Seonghyeon Ye, Doyoung Kim, Sungdong Kim, Hyeonbin Hwang, Seungone Kim, Yongrae Jo, James Thorne, Juho Kim, and Minjoon Seo. 2023. FLASK: Fine-grained Language Model Evaluation based on Alignment Skill Sets. arXiv preprint arXiv:2307.10928 (2023)."},{"key":"e_1_3_3_3_72_1","doi-asserted-by":"publisher","DOI":"10.1145\/3563657.3596138"},{"key":"e_1_3_3_3_73_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581388"},{"key":"e_1_3_3_3_74_1","unstructured":"Renrui Zhang Jiaming Han Chris Liu Peng Gao Aojun Zhou Xiangfei Hu Shilin Yan Pan Lu Hongsheng Li and Yu Qiao. 2023. LLaMA-Adapter: Efficient Fine-tuning of Language Models with Zero-init Attention. arxiv:2303.16199\u00a0[cs.CV]"},{"key":"e_1_3_3_3_75_1","volume-title":"Calibrate Before Use: Improving Few-shot Performance of Language Models. In The 38th International Conference on Machine Learning (ICML \u201921)","author":"Zhao Zihao","year":"2021","unstructured":"Zihao Zhao, Eric Wallace, Shi Feng, Dan Klein, and Sameer Singh. 2021. Calibrate Before Use: Improving Few-shot Performance of Language Models. In The 38th International Conference on Machine Learning (ICML \u201921). 12697\u201312706. http:\/\/proceedings.mlr.press\/v139\/zhao21c.html"},{"key":"e_1_3_3_3_76_1","unstructured":"Lianmin Zheng Wei-Lin Chiang Ying Sheng Siyuan Zhuang Zhanghao Wu Yonghao Zhuang Zi Lin Zhuohan Li Dacheng Li Eric.\u00a0P Xing Hao Zhang Joseph\u00a0E. Gonzalez and Ion Stoica. 2023. Judging LLM-as-a-judge with MT-Bench and Chatbot Arena. arxiv:2306.05685\u00a0[cs.CL]"},{"key":"e_1_3_3_3_77_1","volume-title":"Towards a unified multi-dimensional evaluator for text generation. arXiv preprint arXiv:2210.07197","author":"Zhong Ming","year":"2022","unstructured":"Ming Zhong, Yang Liu, Da Yin, Yuning Mao, Yizhu Jiao, Pengfei Liu, Chenguang Zhu, Heng Ji, and Jiawei Han. 2022. Towards a unified multi-dimensional evaluator for text generation. arXiv preprint arXiv:2210.07197 (2022)."}],"event":{"name":"CHI '24: CHI Conference on Human Factors in Computing Systems","location":"Honolulu HI USA","acronym":"CHI '24","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction","SIGACCESS ACM Special Interest Group on Accessible Computing"]},"container-title":["Proceedings of the CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3613904.3642216","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3613904.3642216","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:44:23Z","timestamp":1750290263000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3613904.3642216"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5,11]]},"references-count":75,"alternative-id":["10.1145\/3613904.3642216","10.1145\/3613904"],"URL":"https:\/\/doi.org\/10.1145\/3613904.3642216","relation":{},"subject":[],"published":{"date-parts":[[2024,5,11]]},"assertion":[{"value":"2024-05-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}