{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T00:10:07Z","timestamp":1755907807048,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":21,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,5,5]],"date-time":"2025-05-05T00:00:00Z","timestamp":1746403200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,5,5]]},"DOI":"10.1145\/3680256.3721312","type":"proceedings-article","created":{"date-parts":[[2025,5,3]],"date-time":"2025-05-03T01:00:58Z","timestamp":1746234058000},"page":"130-135","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["EMU-LLM: Emulators for Performance Evaluation of LLM-based Applications"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-7034-5948","authenticated-orcid":false,"given":"Deeksha","family":"Deeksha","sequence":"first","affiliation":[{"name":"TCS Research, Mumbai, Maharashtra, India"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8592-3132","authenticated-orcid":false,"given":"Ashwin","family":"Krishnan","sequence":"additional","affiliation":[{"name":"TCS Research, Mumbai, Maharashtra, India"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9001-0629","authenticated-orcid":false,"given":"Manoj","family":"Nambiar","sequence":"additional","affiliation":[{"name":"TCS Research, Mumbai, Maharashtra, India"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,5,5]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proceedings of Machine Learning and Systems, P. Gibbons, G. Pekhimenko, and C. De Sa (Eds.)","volume":"6","author":"Agrawal Amey","year":"2024","unstructured":"Amey Agrawal, Nitin Kedia, Jayashree Mohan, Ashish Panwar, Nipun Kwatra, Bhargav Gulavani, Ramachandran Ramjee, and Alexey Tumanov. 2024. VIDUR: A LARGE-SCALE SIMULATION FRAMEWORK FOR LLM INFERENCE. In Proceedings of Machine Learning and Systems, P. Gibbons, G. Pekhimenko, and C. De Sa (Eds.), Vol. 6. 351--366. https:\/\/proceedings.mlsys.org\/paper_files\/paper\/2024\/ file\/b74a8de47d2b3c928360e0a011f48351-Paper-Conference.pdf"},{"key":"e_1_3_2_1_2_1","unstructured":"Anyscale. 2025. Anyscale | Supercharge your AI Platform. https:\/\/www.anyscale. com Accessed: 2025-03--19."},{"key":"e_1_3_2_1_3_1","unstructured":"Apache Software Foundation. 2024. Apache JMeter. https:\/\/jmeter.apache.org\/. Accessed: 2024-09--11."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N18-2097"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2014.01.010"},{"key":"e_1_3_2_1_6_1","unstructured":"Gunicorn Development Team. 2024. Gunicorn. https:\/\/gunicorn.org\/. Accessed: 2024-09--11."},{"key":"e_1_3_2_1_7_1","first-page":"90","volume-title":"P95, and P99. Medium. https:\/\/medium.com\/tuanhdotnet\/statistics-behind-latency-metricsunderstanding-","author":"Tuan H.","year":"2025","unstructured":"Tuan H. 2023. Statistics Behind Latency Metrics: Understanding P90, P95, and P99. Medium. https:\/\/medium.com\/tuanhdotnet\/statistics-behind-latency-metricsunderstanding- p90-p95-and-p99-dc87420d505d Accessed: 2025-03--19."},{"key":"e_1_3_2_1_8_1","unstructured":"Yuchen Eleanor Jiang Tianyu Liu Shuming Ma Dongdong Zhang Mrinmaya Sachan and Ryan Cotterell. 2023. Discourse Centric Evaluation of Machine Translation with a Densely Annotated Parallel Corpus. arXiv:2305.11142 [cs.CL] https:\/\/arxiv.org\/abs\/2305.11142"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/HiPC62374.2024.00018"},{"key":"e_1_3_2_1_10_1","volume-title":"Acceleron: A Tool to Accelerate Research Ideation. arXiv preprint arXiv:2403.04382","author":"Nigam Harshit","year":"2024","unstructured":"Harshit Nigam, Manasi Patwardhan, Lovekesh Vig, and Gautam Shroff. 2024. Acceleron: A Tool to Accelerate Research Ideation. arXiv preprint arXiv:2403.04382 (2024). https:\/\/arxiv.org\/abs\/2403.04382 Accessed: 2024-01--25."},{"key":"e_1_3_2_1_11_1","unstructured":"OpenAI. 2024. OpenAI Platform Documentation. https:\/\/platform.openai.com\/docs\/overview. Accessed: 2024-09--11."},{"volume-title":"Quick start guide to large language models: strategies and best practices for using ChatGPT and other LLMs","author":"Ozdemir Sinan","key":"e_1_3_2_1_12_1","unstructured":"Sinan Ozdemir. 2023. Quick start guide to large language models: strategies and best practices for using ChatGPT and other LLMs. Addison-Wesley Professional."},{"key":"e_1_3_2_1_13_1","volume-title":"The Inference Cost Of Search Disruption-- Large Language Model Cost Analysis. Verf\u00fcgbar unter https:\/\/www. semianalysis.com\/p\/theinference-cost-of-search-disruption","author":"Patel Dylan","year":"2023","unstructured":"Dylan Patel and Afzal Ahmad. 2023. The Inference Cost Of Search Disruption-- Large Language Model Cost Analysis. Verf\u00fcgbar unter https:\/\/www. semianalysis.com\/p\/theinference-cost-of-search-disruption (2023)."},{"key":"e_1_3_2_1_14_1","unstructured":"Pinecone. 2024. Pinecone Documentation. https:\/\/docs.pinecone.io\/home. Accessed: 2024-09--11."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1000\/jair.2023.12345"},{"key":"e_1_3_2_1_16_1","unstructured":"Vanna.AI Team. 2024. Vanna.AI: Personalized AI SQL Agent. https:\/\/vannaai.com. Accessed: 2024-01--25."},{"key":"e_1_3_2_1_17_1","unstructured":"Together AI. 2024. Introduction to Together AI Documentation. https:\/\/docs. together.ai\/docs\/introduction. Accessed: 2024-09--11."},{"key":"e_1_3_2_1_18_1","unstructured":"Tricentis. 2024. NeoLoad Trial. https:\/\/www.tricentis.com\/software-testing-tooltrial- demo\/neoload-trial. Accessed: 2024-09--11."},{"key":"e_1_3_2_1_19_1","unstructured":"Vercel. 2025. Fireworks \/ Anyscale \/ Together docs pages (for their API configs). https:\/\/github.com\/vercel\/modelfusion\/issues\/243 Accessed: 2025-03--19."},{"key":"e_1_3_2_1_20_1","volume-title":"Harnessing the power of llms in practice: A survey on chatgpt and beyond. ACM Transactions on Knowledge Discovery from Data 18, 6","author":"Yang Jingfeng","year":"2024","unstructured":"Jingfeng Yang, Hongye Jin, Ruixiang Tang, Xiaotian Han, Qizhang Feng, Haoming Jiang, Shaochen Zhong, Bing Yin, and Xia Hu. 2024. Harnessing the power of llms in practice: A survey on chatgpt and beyond. ACM Transactions on Knowledge Discovery from Data 18, 6 (2024), 1--32."},{"key":"e_1_3_2_1_21_1","unstructured":"Lianmin Zheng Wei-Lin Chiang Ying Sheng Tianle Li Siyuan Zhuang Zhanghao Wu Yonghao Zhuang Zhuohan Li Zi Lin Eric. P Xing Joseph E. Gonzalez Ion Stoica and Hao Zhang. 2023. LMSYS-Chat-1M: A Large-Scale Real-World LLM Conversation Dataset. arXiv:2309.11998 [cs.CL]"}],"event":{"name":"ICPE '25: 16th ACM\/SPEC International Conference on Performance Engineering","sponsor":["SIGMETRICS ACM Special Interest Group on Measurement and Evaluation","SIGSOFT ACM Special Interest Group on Software Engineering"],"location":"Toronto ON Canada","acronym":"ICPE '25"},"container-title":["Companion of the 16th ACM\/SPEC International Conference on Performance Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3680256.3721312","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3680256.3721312","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T23:49:32Z","timestamp":1755906572000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3680256.3721312"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,5,5]]},"references-count":21,"alternative-id":["10.1145\/3680256.3721312","10.1145\/3680256"],"URL":"https:\/\/doi.org\/10.1145\/3680256.3721312","relation":{},"subject":[],"published":{"date-parts":[[2025,5,5]]},"assertion":[{"value":"2025-05-05","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}