{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,13]],"date-time":"2026-02-13T09:51:19Z","timestamp":1770976279180,"version":"3.50.1"},"publisher-location":"Singapore","reference-count":36,"publisher":"Springer Nature Singapore","isbn-type":[{"value":"9789819698110","type":"print"},{"value":"9789819698127","type":"electronic"}],"license":[{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-981-96-9812-7_42","type":"book-chapter","created":{"date-parts":[[2025,7,25]],"date-time":"2025-07-25T07:27:34Z","timestamp":1753428454000},"page":"510-520","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Hard400: A Bilingual Code Generation Evaluation Benchmark for Large Language Models"],"prefix":"10.1007","author":[{"given":"Qin","family":"Zhang","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Song","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Junjie","family":"Li","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hong","family":"Zhou","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xiaojun","family":"Chen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Han","family":"Liu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2025,7,26]]},"reference":[{"issue":"4","key":"42_CR1","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3212695","volume":"51","author":"M Allamanis","year":"2018","unstructured":"Allamanis, M., Barr, E.T., Devanbu, P., Sutton, C.: A survey of machine learning for big code and naturalness. ACM Comput. Surv. (CSUR) 51(4), 1\u201337 (2018)","journal-title":"ACM Comput. Surv. (CSUR)"},{"key":"42_CR2","unstructured":"Athiwaratkun, B., et al.: Multi-lingual evaluation of code generation models. arXiv preprint arXiv:2210.14868 (2022)"},{"key":"42_CR3","unstructured":"Austin, J., et al.: Program synthesis with large language models. arXiv preprint arXiv:2108.07732 (2021)"},{"key":"42_CR4","unstructured":"Bai, J., et al.: Qwen technical report. arXiv preprint arXiv:2309.16609 (2023)"},{"key":"42_CR5","unstructured":"Cai, Z., et al.: InternLM2 technical report. arXiv preprint arXiv:2403.17297 (2024)"},{"key":"42_CR6","doi-asserted-by":"crossref","unstructured":"Cambronero, J., et al.: FlashFill++: scaling programming by example by cutting to the chase. In: Proceedings of the ACM on Programming Languages, vol. 7, no. POPL, pp. 952\u2013981 (2023)","DOI":"10.1145\/3571226"},{"key":"42_CR7","unstructured":"Cassano, F., et al.: A scalable and extensible approach to benchmarking NL2code for 18 programming languages. arXiv preprint arXiv:2208.08227 (2022)"},{"key":"42_CR8","doi-asserted-by":"crossref","unstructured":"Chaudhuri, S., et al.: Neurosym-bolicprogramming. Found. Trends\u00ae Program. Lang. 7(3), 158\u2013243 (2021)","DOI":"10.1561\/2500000049"},{"key":"42_CR9","unstructured":"Chen, M., et al.: Evaluating large language models trained on code. arXiv preprint arXiv:2107.03374 (2021)"},{"key":"42_CR10","unstructured":"Du, Z., et al.: GLM: general language model pretraining with autoregressive blank infilling. arXiv preprint arXiv:2103.10360 (2021)"},{"key":"42_CR11","doi-asserted-by":"publisher","first-page":"111741","DOI":"10.1016\/j.jss.2023.111741","volume":"203","author":"M Evtikhiev","year":"2023","unstructured":"Evtikhiev, M., Bogomolov, E., Sokolov, Y., Bryksin, T.: Out of the bleu: how should we assess quality of the code generation models? J. Syst. Softw. 203, 111741 (2023)","journal-title":"J. Syst. Softw."},{"key":"42_CR12","unstructured":"Fried, D., et al.: Incoder: a generative model for code infilling and synthesis. arXiv preprint arXiv:2204.05999 (2022)"},{"key":"42_CR13","unstructured":"Fu, L., et al.: CodeApex: a bilingual programming evaluation benchmark for large language models. arXiv preprint arXiv:2309.01940 (2023)"},{"key":"42_CR14","doi-asserted-by":"crossref","unstructured":"Gulwani, S., Polozov, O., Singh, R.: Program synthesis. Found. Trends\u00ae Program. Lang. 4(1\u20132), 1\u2013119 (2017)","DOI":"10.1561\/2500000010"},{"key":"42_CR15","unstructured":"Hao, Y., et al.: AixBench: a code generation benchmark dataset. ArXiv abs\/2206.13179 (2022). https:\/\/api.semanticscholar.org\/CorpusID:250072468"},{"key":"42_CR16","unstructured":"Hendrycks, D., et al.: Measuring coding challenge competence with PPS. arXiv preprint arXiv:2105.09938 (2021)"},{"key":"42_CR17","unstructured":"Huang, Y., et al.: C-eval: a multi-level multi-discipline chinese evaluation suite for foundation models. arXiv preprint arXiv:2305.08322 (2023)"},{"key":"42_CR18","unstructured":"Husain, H., Wu, H.H., Gazit, T., Allamanis, M., Brockschmidt, M.: CodeSearchNet challenge: evaluating the State of Semantic Code Search. arXiv preprint arXiv:1909.09436 (2019)"},{"key":"42_CR19","unstructured":"Isabelle, P., Charniak, E., Lin, D.: Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics. In: Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics (2002)"},{"key":"42_CR20","unstructured":"Jiang, A.Q., et al.: Mistral7B. arXiv preprint arXiv:2310.06825 (2023)"},{"key":"42_CR21","unstructured":"Lai, Y., et al.: Ds-1000: a natural and reliable benchmark for data science code generation. In: International Conference on Machine Learning, pp. 18319\u201318345. PMLR (2023)"},{"key":"42_CR22","unstructured":"Li, R., et al.: StarCoder: may the source be with you! arXiv preprint arXiv:2305.06161 (2023)"},{"key":"42_CR23","unstructured":"Lu, S., et al.: Codexglue: a machine learning benchmark dataset for code understanding and generation. arXiv preprint arXiv:2102.04664 (2021)"},{"key":"42_CR24","unstructured":"Luo, Z., et al.: WizardCoder: empowering code large language models with evol-instruct. arXiv preprint arXiv:2306.08568 (2023)"},{"key":"42_CR25","unstructured":"OpenAI, Achiam, J., Adler, S., Agarwal, S., Ahmad, L., Akkaya, I.: GPT-4 technical report (2023)"},{"key":"42_CR26","unstructured":"Press, O., Smith, N.A., Lewis, M.: Train short, test long: attention with linear biases enables input length extrapolation. arXiv preprint arXiv:2108.12409 (2021)"},{"key":"42_CR27","unstructured":"Roziere, B., et al.: Code LLaMA: open foundation models for code. arXiv preprint arXiv:2308.12950 (2023)"},{"key":"42_CR28","unstructured":"Sun, T., et al.: MOSS: training conversational language models from synthetic data. arXiv preprint arXiv:2307.15020 7 (2023)"},{"key":"42_CR29","unstructured":"Team, G., et al.: Gemma: open models based on Gemini research and technology. arXiv preprint arXiv:2403.08295 (2024)"},{"key":"42_CR30","unstructured":"Touvron, H., et al.: LLaMA 2: open foundation and fine-tuned chat models. arXiv preprint arXiv:2307.09288 (2023)"},{"key":"42_CR31","unstructured":"Yang, A., et al.: Baichuan2: open large-scale language models. arXiv preprint arXiv:2309.10305 (2023)"},{"key":"42_CR32","doi-asserted-by":"crossref","unstructured":"Yin, P., Neubig, G.: A syntactic neural model for general-purpose code generation. arXiv preprint arXiv:1704.01696 (2017)","DOI":"10.18653\/v1\/P17-1041"},{"key":"42_CR33","unstructured":"Young, A., et al.: Yi: open foundation models by 01.AI. arXiv preprint arXiv:2403.04652 (2024)"},{"key":"42_CR34","doi-asserted-by":"crossref","unstructured":"Zan, D., et al.: When language model meets private library. arXiv preprint arXiv:2210.17236 (2022)","DOI":"10.18653\/v1\/2022.findings-emnlp.21"},{"key":"42_CR35","doi-asserted-by":"crossref","unstructured":"Zan, D., et al.: CERT: continual pre-training on sketches for library-oriented code generation. arXiv preprint arXiv:2206.06888 (2022)","DOI":"10.24963\/ijcai.2022\/329"},{"key":"42_CR36","doi-asserted-by":"crossref","unstructured":"Zheng, Q., et al.: CodeGeeX: a pre-trained model for code generation with multilingual evaluations on HumanEval-X. arXiv preprint arXiv:2303.17568 (2023)","DOI":"10.1145\/3580305.3599790"}],"container-title":["Lecture Notes in Computer Science","Advanced Intelligent Computing Technology and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-981-96-9812-7_42","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,2,13]],"date-time":"2026-02-13T08:57:20Z","timestamp":1770973040000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-981-96-9812-7_42"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"ISBN":["9789819698110","9789819698127"],"references-count":36,"URL":"https:\/\/doi.org\/10.1007\/978-981-96-9812-7_42","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025]]},"assertion":[{"value":"26 July 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ICIC","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Intelligent Computing","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Ningbo","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"China","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2025","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"26 July 2025","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 July 2025","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"21","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"icic2025","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/www.ic-icc.cn\/icg\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}