{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,4]],"date-time":"2026-05-04T06:33:56Z","timestamp":1777876436601,"version":"3.51.4"},"publisher-location":"Cham","reference-count":20,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032180698","type":"print"},{"value":"9783032180704","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-3-032-18070-4_13","type":"book-chapter","created":{"date-parts":[[2026,4,30]],"date-time":"2026-04-30T22:23:09Z","timestamp":1777587789000},"page":"203-220","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Next Generative AI Benchmark Development"],"prefix":"10.1007","author":[{"given":"Hamesh","family":"Patel","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nirmala","family":"Sundararajan","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nicholas","family":"Wakou","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Paul","family":"Cao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"David","family":"Schmidt","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2026,4,1]]},"reference":[{"key":"13_CR1","unstructured":"Amini, A., Vieira, T., Cotterell, R.: Better estimation of the kl divergence between language models (2025). https:\/\/arxiv.org\/abs\/2504.10637"},{"key":"13_CR2","unstructured":"Anil, R., et al.: Gemini: a family of highly capable multimodal models (2025). https:\/\/arxiv.org\/abs\/2312.11805"},{"key":"13_CR3","unstructured":"Chen, C., Borgeaud, S., Irving, G., Lespiau, J.B., Sifre, L., Jumper, J.: Accelerating large language model decoding with speculative sampling (2023). https:\/\/arxiv.org\/abs\/2302.01318"},{"issue":"8053","key":"13_CR4","doi-asserted-by":"publisher","first-page":"22","DOI":"10.1038\/d41586-025-00616-z","volume":"639","author":"S Chen","year":"2025","unstructured":"Chen, S.: How much energy will AI really consume? the good, the bad and the unknown. Nature 639(8053), 22\u201324 (2025)","journal-title":"Nature"},{"key":"13_CR5","doi-asserted-by":"publisher","first-page":"16344","DOI":"10.52202\/068431-1189","volume":"35","author":"T Dao","year":"2022","unstructured":"Dao, T., Fu, D., Ermon, S., Rudra, A., R\u00e9, C.: Flashattention: fast and memoryefficient exact attention with IO-awareness. Adv. Neural. Inf. Process. Syst. 35, 16344\u201316359 (2022)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"13_CR6","unstructured":"Hendrycks, D., et al.: Measuring massive multitask language understanding (2021). https:\/\/arxiv.org\/abs\/2009.03300"},{"key":"13_CR7","unstructured":"Jegham, N., Abdelatti, M., Elmoubarki, L., Hendawi, A.: How hungry is AI? benchmarking energy, water, and carbon footprint of LLM inference (2025). https:\/\/arxiv.org\/abs\/2505.09598"},{"key":"13_CR8","unstructured":"Kaplan, J., et al.: Scaling laws for neural language models (2020). https:\/\/arxiv.org\/abs\/2001.08361"},{"key":"13_CR9","unstructured":"Khattab, O., et al.: Dspy: compiling declarative language model calls into self-improving pipelines (2024)"},{"key":"13_CR10","doi-asserted-by":"publisher","unstructured":"Kwon, W., et al.: Efficient memory management for large language model serving with paged attention. In: Proceedings of the 29th Symposium on Operating Systems Principles, SOSP 2023, p. 611626. Association for Computing Machinery, New York (2023). https:\/\/doi.org\/10.1145\/3600006.3613165","DOI":"10.1145\/3600006.3613165"},{"key":"13_CR11","unstructured":"Leviathan, Y., Kalman, M., Matias, Y.: Fast inference from transformers via speculative decoding. In: Krause, A., Brunskill, E., Cho, K., Engelhardt, B., Sabato, S., Scarlett, J. (eds.) Proceedings of the 40th International Conference on Machine Learning. Proceedings of Machine Learning Research, vol. 202, pp. 19274\u201319286. PMLR (2023). https:\/\/proceedings.mlr.press\/v202\/leviathan23a.html"},{"key":"13_CR12","unstructured":"Liang, P., et al.: Holistic evaluation of language models (2023). https:\/\/arxiv.org\/abs\/2211.09110"},{"key":"13_CR13","unstructured":"Mai, Y., Liang, P.: Massive multitask language understanding (mmlu) on helm (2024). https:\/\/crfm.stanford.edu\/2024\/02\/15\/mmlu.html"},{"key":"13_CR14","unstructured":"Otten, N.V.: Perplexity in NLP: understand how to evaluate LLMs (2024). https:\/\/spotintelligence.com\/2024\/08\/19\/perplexity-in-nlp\/"},{"key":"13_CR15","unstructured":"Pangakis, N., Wolken, S.: Keeping humans in the loop: human-centered automated annotation with generative AI (2024). https:\/\/arxiv.org\/abs\/2409.09467"},{"key":"13_CR16","unstructured":"Pope, R., et al.: Efficiently scaling transformer inference (2022). https:\/\/arxiv.org\/abs\/2211.05102"},{"key":"13_CR17","unstructured":"Tunguz, T.: AI pricing: How much does AI cost in 2025? (2025). https:\/\/www.getmonetizely.com\/blogs\/ai-pricing-how-much-does-ai-cost-in-2025"},{"key":"13_CR18","unstructured":"Vaswani, A., et al.: Attention is all you need. In: Proceedings of the 31st International Conference on Neural Information Processing Systems, NIPS 2017, p. 60006010. Curran Associates Inc., Red Hook (2017)"},{"key":"13_CR19","doi-asserted-by":"publisher","unstructured":"Williams, A.: Comparison of generative AI performance on undergraduate and postgraduate written assessments in the biomedical sciences. Int. J. Educ. Technol. High. Educ. 21 (2024). https:\/\/doi.org\/10.1186\/s41239-024-00485-y","DOI":"10.1186\/s41239-024-00485-y"},{"key":"13_CR20","unstructured":"Zellinger, M.J., Thomson, M.: Economic evaluation of LLMs (2025). https:\/\/arxiv.org\/abs\/2507.03834"}],"container-title":["Lecture Notes in Computer Science","Performance Evaluation and Benchmarking"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-18070-4_13","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,30]],"date-time":"2026-04-30T22:23:11Z","timestamp":1777587791000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-18070-4_13"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"ISBN":["9783032180698","9783032180704"],"references-count":20,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-18070-4_13","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]},"assertion":[{"value":"1 April 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"TPCTC","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Technology Conference on Performance Evaluation and Benchmarking","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"London","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"United Kingdom","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2025","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1 September 2025","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1 September 2025","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"17","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"tpctc2025","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.tpc.org\/tpctc\/tpctc2025\/default5.asp","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}