{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,9]],"date-time":"2026-07-09T18:47:53Z","timestamp":1783622873639,"version":"3.55.0"},"reference-count":254,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2025,11,20]],"date-time":"2025-11-20T00:00:00Z","timestamp":1763596800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"},{"start":{"date-parts":[[2025,11,20]],"date-time":"2025-11-20T00:00:00Z","timestamp":1763596800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["DMS-2101104"],"award-info":[{"award-number":["DMS-2101104"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["DMS-1925066"],"award-info":[{"award-number":["DMS-1925066"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["DMS-1925066"],"award-info":[{"award-number":["DMS-1925066"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100005246","name":"Institute of Education Sciences","doi-asserted-by":"publisher","award":["R305C240010"],"award-info":[{"award-number":["R305C240010"]}],"id":[{"id":"10.13039\/100005246","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R01GM152814"],"award-info":[{"award-number":["R01GM152814"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R01GM152814"],"award-info":[{"award-number":["R01GM152814"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Artif Intell Rev"],"DOI":"10.1007\/s10462-025-11423-3","type":"journal-article","created":{"date-parts":[[2025,11,20]],"date-time":"2025-11-20T05:20:38Z","timestamp":1763616038000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":31,"title":["Knowledge distillation and dataset distillation of large language models: emerging trends, challenges, and future directions"],"prefix":"10.1007","volume":"59","author":[{"given":"Luyang","family":"Fang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaowei","family":"Yu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiazhang","family":"Cai","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yongkai","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shushan","family":"Wu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhengliang","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhenyuan","family":"Yang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Haoran","family":"Lu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xilin","family":"Gong","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yufang","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Terry","family":"Ma","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wei","family":"Ruan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ali","family":"Abbasi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jing","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tao","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ehsan","family":"Latif","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wei","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wei","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Soheil","family":"Kolouri","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaoming","family":"Zhai","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Dajiang","family":"Zhu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wenxuan","family":"Zhong","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tianming","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ping","family":"Ma","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2025,11,20]]},"reference":[{"key":"11423_CR1","unstructured":"Abbas A, Tirumala K, Simig D, Ganguli S, Morcos AS (2023) Semdedup: Data-efficient learning at web-scale through semantic deduplication. arXiv preprint arXiv:2303.09540"},{"key":"11423_CR2","unstructured":"Albalak A, Elazar Y, Xie SM, Longpre S, Lambert N, Wang X, Muennighoff N, Hou B, Pan L, Jeong H, et al (2024) A survey on data selection for language models. Trans Mach Learn Res"},{"key":"11423_CR3","unstructured":"Allen-Zhu Z, Li Y (2020) Towards understanding ensemble, knowledge distillation and self-distillation in deep learning. arXiv preprint arXiv:2012.09816"},{"key":"11423_CR5","doi-asserted-by":"crossref","unstructured":"Arazo E, Ortego D, Albert P, O\u2019Connor NE, McGuinness K (2020) Pseudo-labeling and confirmation bias in deep semi-supervised learning. In: 2020 International joint conference on neural networks (IJCNN). IEEE, pp 1\u20138","DOI":"10.1109\/IJCNN48605.2020.9207304"},{"key":"11423_CR7","doi-asserted-by":"crossref","unstructured":"Augenstein I, Baldwin T, Cha M, Chakraborty T, Ciampaglia GL, Corney D, DiResta R, Ferrara E, Hale S, Halevy A et al (2023) Factuality challenges in the era of large language models. arXiv preprint arXiv:2310.05189","DOI":"10.1038\/s42256-024-00881-z"},{"issue":"8","key":"11423_CR6","doi-asserted-by":"publisher","first-page":"852","DOI":"10.1038\/s42256-024-00881-z","volume":"6","author":"I Augenstein","year":"2024","unstructured":"Augenstein I, Baldwin T, Cha M, Chakraborty T, Ciampaglia GL, Corney D, DiResta R, Ferrara E, Hale S, Halevy A et al (2024) Factuality challenges in the era of large language models and opportunities for fact-checking. Nat Mach Intell 6(8):852\u2013863","journal-title":"Nat Mach Intell"},{"key":"11423_CR8","unstructured":"Axelrod A, He X, Gao J (2011) Domain adaptation via pseudo in-domain data selection. In: Proceedings of the 2011 conference on empirical methods in natural language processing, pp 355\u2013362"},{"key":"11423_CR9","doi-asserted-by":"crossref","unstructured":"Balad\u00f3n A, Sastre I, Chiruzzo L, Ros\u00e1 A (2023) Retuyt-inco at bea 2023 shared task: Tuning open-source llms for generating teacher responses. In: Proceedings of the 18th workshop on innovative use of NLP for building educational applications (BEA 2023), pp 756\u2013765","DOI":"10.18653\/v1\/2023.bea-1.61"},{"key":"11423_CR10","doi-asserted-by":"crossref","unstructured":"Bender EM, Gebru T, McMillan-Major A, Shmitchell S (2021) On the dangers of stochastic parrots: can language models be too big? In: Proceedings of the 2021 ACM conference on fairness, accountability, and transparency, pp 610\u2013623","DOI":"10.1145\/3442188.3445922"},{"key":"11423_CR11","doi-asserted-by":"crossref","unstructured":"Binici K, Pham NT, Mitra T, Leman K (2022) Preventing catastrophic forgetting and distribution mismatch in knowledge distillation via synthetic data. In: Proceedings of the IEEE\/CVF winter conference on applications of computer vision, pp 663\u2013671","DOI":"10.1109\/WACV51458.2022.00368"},{"issue":"518","key":"11423_CR12","doi-asserted-by":"publisher","first-page":"859","DOI":"10.1080\/01621459.2017.1285773","volume":"112","author":"DM Blei","year":"2017","unstructured":"Blei DM, Kucukelbir A, McAuliffe JD (2017) Variational inference: a review for statisticians. J Am Stat Assoc 112(518):859\u2013877","journal-title":"J Am Stat Assoc"},{"key":"11423_CR13","doi-asserted-by":"crossref","unstructured":"Blowe R, Vandersteen A, Winterbourne D, Hathersage J, Ravenscroft M (2024) Semantic entanglement modeling for knowledge distillation in large language models. Authorea Preprints","DOI":"10.36227\/techrxiv.173337599.98527860\/v1"},{"key":"11423_CR14","first-page":"14879","volume":"33","author":"Z Borsos","year":"2020","unstructured":"Borsos Z, Mutny M, Krause A (2020) Coresets via bilevel optimization for continual learning and streaming. Adv Neural Inf Process Syst 33:14879\u201314890","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR15","first-page":"5316","volume":"34","author":"K Borup","year":"2021","unstructured":"Borup K, Andersen LN (2021) Even your teacher needs guidance: ground-truth targets dampen regularization imposed by self-distillation. Adv Neural Inf Process Syst 34:5316\u20135327","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR16","first-page":"1877","volume":"33","author":"T Brown","year":"2020","unstructured":"Brown T, Mann B, Ryder N, Subbiah M, Kaplan JD, Dhariwal P, Neelakantan A, Shyam P, Sastry G, Askell A et al (2020) Language models are few-shot learners. Adv Neural Inf Process Syst 33:1877\u20131901","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR17","doi-asserted-by":"crossref","unstructured":"Cazenavette G, Wang T, Torralba A, Efros AA, Zhu J-Y (2022) Dataset distillation by matching training trajectories. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 4750\u20134759","DOI":"10.1109\/CVPR52688.2022.01045"},{"key":"11423_CR18","doi-asserted-by":"crossref","unstructured":"Cazenavette G, Wang T, Torralba A, Efros AA, Zhu J-Y (2023) Generalizing dataset distillation via deep generative prior. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 3739\u20133748","DOI":"10.1109\/CVPR52729.2023.00364"},{"key":"11423_CR20","unstructured":"Chang X, Lee S.Y.M, Zhu S, Li S, Zhou G (2022) One-teacher and multiple-student knowledge distillation on sentiment classification. In: Calzolari N, Huang C-R, Kim H, Pustejovsky J, Wanner L, Choi K-S, Ryu P-M, Chen H-H, Donatelli L, Ji H, Kurohashi S, Paggio P, Xue N, Kim S, Hahm Y, He Z, Lee TK, Santus E, Bond F, Na S-H (eds.) Proceedings of the 29th international conference on computational linguistics. International Committee on Computational Linguistics, Gyeongju, Republic of Korea, pp 7042\u20137052. https:\/\/aclanthology.org\/2022.coling-1.614\/"},{"issue":"3","key":"11423_CR19","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3641289","volume":"15","author":"Y Chang","year":"2024","unstructured":"Chang Y, Wang X, Wang J, Wu Y, Yang L, Zhu K, Chen H, Yi X, Wang C, Wang Y et al (2024) A survey on evaluation of large language models. ACM Trans Intell Syst Tech 15(3):1\u201345","journal-title":"ACM Trans Intell Syst Tech"},{"key":"11423_CR21","unstructured":"Chen JC-Y, Saha S, Stengel-Eskin E, Bansal M (2024) Magdi: Structured distillation of multi-agent interaction graphs improves reasoning in smaller language models. arXiv preprint arXiv:2402.01620"},{"key":"11423_CR22","unstructured":"Chiang W-L, Li Z, Lin Z, Sheng Y, Wu Z, Zhang H, Zheng L, Zhuang S, Zhuang Y, Gonzalez JE et al (2023) Vicuna: An open-source chatbot impressing gpt-4 with 90%* chatgpt quality. See https:\/\/vicunalmsys.org (accessed 14 April 2023) 2(3), 6"},{"key":"11423_CR23","unstructured":"Chu Z, Chen J, Chen Q, Yu W, He T, Wang H, Peng W, Liu M, Qin B, Liu T (2023) A survey of chain of thought reasoning: advances, frontiers and future. arXiv preprint arXiv:2309.15402"},{"key":"11423_CR24","unstructured":"Cobbe K, Kosaraju V, Bavarian M, Chen M, Jun H, Kaiser L, Plappert M, Tworek J, Hilton J, Nakano R et al (2021) Training verifiers to solve math word problems. arXiv preprint arXiv:2110.14168"},{"issue":"11","key":"11423_CR26","doi-asserted-by":"publisher","first-page":"0000651","DOI":"10.1371\/journal.pdig.0000651","volume":"3","author":"JL Cross","year":"2024","unstructured":"Cross JL, Choma MA, Onofrey JA (2024) Bias in medical ai: implications for clinical decision-making. PLOS Digital Health 3(11):0000651","journal-title":"PLOS Digital Health"},{"key":"11423_CR27","unstructured":"Cui J, Wang R, Si S, Hsieh C-J (2023) Scaling up dataset distillation to imagenet-1k with constant memory. In: ICML, pp 6565\u20136590"},{"issue":"1","key":"11423_CR28","doi-asserted-by":"publisher","first-page":"1418","DOI":"10.1038\/s41467-024-45563-x","volume":"15","author":"J Dagdelen","year":"2024","unstructured":"Dagdelen J, Dunn A, Lee S, Walker N, Rosen AS, Ceder G, Persson KA, Jain A (2024) Structured information extraction from scientific text with large language models. Nat Commun 15(1):1418","journal-title":"Nat Commun"},{"key":"11423_CR29","unstructured":"Dan Y, Lei Z, Gu Y, Li Y, Yin J, Lin J, Ye L, Tie Z, Zhou Y, Wang Y et al (2023) Educhat: A large-scale language model-based chatbot system for intelligent education. arXiv preprint arXiv:2308.02773"},{"issue":"5","key":"11423_CR30","doi-asserted-by":"publisher","first-page":"2060","DOI":"10.1137\/070696507","volume":"38","author":"A Dasgupta","year":"2009","unstructured":"Dasgupta A, Drineas P, Harb B, Kumar R, Mahoney MW (2009) Sampling algorithms and coresets for $$\\backslash $$ell_p regression. SIAM J Comput 38(5):2060\u20132078","journal-title":"SIAM J Comput"},{"key":"11423_CR31","first-page":"34391","volume":"35","author":"Z Deng","year":"2022","unstructured":"Deng Z, Russakovsky O (2022) Remember the past: distilling datasets into addressable memories for neural networks. Adv Neural Inf Process Syst 35:34391\u201334404","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR32","doi-asserted-by":"crossref","unstructured":"Deng M, Wang J, Hsieh C-P, Wang Y, Guo H, Shu T, Song M, Xing EP, Hu Z (2022) Rlprompt: Optimizing discrete text prompts with reinforcement learning. arXiv preprint arXiv:2205.12548","DOI":"10.18653\/v1\/2022.emnlp-main.222"},{"key":"11423_CR33","unstructured":"DeSalvo G, Kagy J-F, Karydas L, Rostamizadeh A, Kumar S (2024) No more hard prompts: Softsrv prompting for synthetic data generation. arXiv preprint arXiv:2410.16534"},{"key":"11423_CR34","unstructured":"Ding M, Xu Y, Rabbani T, Liu X, Gravelle B, Ranadive T, Tuan T-C, Huang F (2024a) Calibrated dataset condensation for faster hyperparameter search. arXiv preprint arXiv:2405.17535"},{"key":"11423_CR35","doi-asserted-by":"crossref","unstructured":"Ding S, Ye J, Hu X, Zou N (2024b) Distilling the knowledge from large-language model for health event prediction. arXiv preprint arXiv:2403.03242","DOI":"10.1101\/2024.06.23.24309365"},{"key":"11423_CR36","unstructured":"Dong Y, Mu R, Jin G, Qi Y, Hu J, Zhao X, Meng J, Ruan W, Huang X (2024a) Building guardrails for large language models. arXiv preprint arXiv:2402.01822"},{"key":"11423_CR37","doi-asserted-by":"crossref","unstructured":"Dong Y, Mu R, Zhang Y, Sun S, Zhang T, Wu C, Jin G, Qi Y, Hu J, Meng J et al (2024b) Safeguarding large language models: a survey. arXiv preprint arXiv:2406.02622","DOI":"10.1007\/s10462-025-11389-2"},{"key":"11423_CR38","first-page":"12345","volume":"33","author":"S Du","year":"2020","unstructured":"Du S, You S, Li X, Wu J, Wang F, Qian C, Zhang C (2020) Agree to disagree: adaptive ensemble knowledge distillation in gradient space. Adv Neural Inf Process Syst 33:12345\u201312355","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR40","unstructured":"Fang L, Chen Y, Zhong W, Ma P (2024) Bayesian knowledge distillation: a Bayesian perspective of distillation with uncertainty quantification. In: Forty-first international conference on machine learning"},{"issue":"5","key":"11423_CR39","doi-asserted-by":"publisher","first-page":"1060","DOI":"10.26599\/BDMA.2025.9020011","volume":"8","author":"L Fang","year":"2025","unstructured":"Fang L, Meng C, Zhao L, Wang T, Liu T, Zhong W, Ma P (2025a) Spot: an active learning algorithm for efficient deep neural network training. Big Data Mining Anal 8(5):1060\u20131074","journal-title":"Big Data Mining Anal"},{"key":"11423_CR41","doi-asserted-by":"crossref","unstructured":"Fang L, Latif E, Lu H, Zhou Y, Ma P, Zhai X (2025b) Efficient multi-task inferencing: model merging with Gromov-Wasserstein feature alignment. arXiv preprint arXiv:2503.09774","DOI":"10.1007\/978-3-031-99267-4_24"},{"issue":"120","key":"11423_CR42","first-page":"1","volume":"23","author":"W Fedus","year":"2022","unstructured":"Fedus W, Zoph B, Shazeer N (2022) Switch transformers: scaling to trillion parameter models with simple and efficient sparsity. J Mach Learn Res 23(120):1\u201339","journal-title":"J Mach Learn Res"},{"key":"11423_CR43","unstructured":"Feng K, Li C, Zhang X, Zhou J, Yuan Y, Wang G (2024) Keypoint-based progressive chain-of-thought distillation for llms. arXiv preprint arXiv:2405.16064"},{"key":"11423_CR44","doi-asserted-by":"crossref","unstructured":"Fukuda T, Suzuki M, Kurata G, Thomas S, Cui J, Ramabhadran B (2017) Efficient knowledge distillation from an ensemble of teachers. In: Interspeech, pp 3697\u20133701","DOI":"10.21437\/Interspeech.2017-614"},{"key":"11423_CR45","unstructured":"Gao L, Biderman S, Black S, Golding L, Hoppe T, Foster C, Phang J, He H, Thite A, Nabeshima N et al (2020) The pile: an 800gb dataset of diverse text for language modeling. arXiv preprint arXiv:2101.00027"},{"key":"11423_CR46","unstructured":"Ge H, Hao L, Xu Z et al (2025) Clinkd: Cross-modal clinic knowledge distiller for multi-task medical images. arXiv preprint arXiv:2503.01034"},{"key":"11423_CR47","unstructured":"Ghorbani A, Zou J (2019) Data shapley: Equitable valuation of data for machine learning. In: International conference on machine learning. PMLR, pp 2242\u20132251"},{"issue":"6","key":"11423_CR48","doi-asserted-by":"publisher","first-page":"1789","DOI":"10.1007\/s11263-021-01453-z","volume":"129","author":"J Gou","year":"2021","unstructured":"Gou J, Yu B, Maybank SJ, Tao D (2021) Knowledge distillation: a survey. Int J Comput Vis 129(6):1789\u20131819","journal-title":"Int J Comput Vis"},{"key":"11423_CR49","unstructured":"Gu J, Bradbury J, Xiong C, Li VO, Socher R (2017) Non-autoregressive neural machine translation. arXiv preprint arXiv:1711.02281"},{"key":"11423_CR50","unstructured":"Guo C, Pleiss G, Sun Y, Weinberger KQ (2017) On calibration of modern neural networks. In: International conference on machine learning. PMLR, pp 1321\u20131330"},{"key":"11423_CR51","unstructured":"Guo D, Yang D, Zhang H, Song J, Zhang R, Xu R, Zhu Q, Ma S, Wang P, Bi X et al (2025) Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning. arXiv preprint arXiv:2501.12948"},{"key":"11423_CR52","doi-asserted-by":"crossref","unstructured":"Gururangan S, Marasovi\u0107 A, Swayamdipta S, Lo K, Beltagy I, Downey D, Smith NA (2020) Don\u2019t stop pretraining: adapt language models to domains and tasks. arXiv preprint arXiv:2004.10964","DOI":"10.18653\/v1\/2020.acl-main.740"},{"key":"11423_CR53","unstructured":"Gu Y, Zhang S, Usuyama N et al (2023) Distilling large language models for biomedical knowledge extraction: a case study on adverse drug events. arXiv preprint arXiv:2311.09602"},{"key":"11423_CR54","doi-asserted-by":"crossref","unstructured":"Hadi MU, Qureshi R, Shah A, Irfan M, Zafar A, Shaikh M.B, Akhtar N, Wu J, Mirjalili S et al (2023) A survey on large language models: applications, challenges, limitations, and practical usage. Authorea Preprints 3","DOI":"10.36227\/techrxiv.23589741.v1"},{"key":"11423_CR55","doi-asserted-by":"crossref","unstructured":"Hasan MJ, Rahman F, Mohammed N (2024) Optimclm: Optimizing clinical language models for predicting patient outcomes. arXiv preprint arXiv:2403.02559","DOI":"10.2139\/ssrn.4797716"},{"key":"11423_CR56","doi-asserted-by":"publisher","first-page":"826","DOI":"10.1162\/tacl_a_00492","volume":"10","author":"X He","year":"2022","unstructured":"He X, Nassar I, Kiros J, Haffari G, Norouzi M (2022) Generate, annotate, and learn: Nlp with synthetic text. Trans Assoc Comput Linguist 10:826\u2013842","journal-title":"Trans Assoc Comput Linguist"},{"key":"11423_CR57","doi-asserted-by":"crossref","unstructured":"He J, Guo H, Zhu K, Zhao Z, Tang M, Wang J (2024) Seekr: Selective attention-guided knowledge retention for continual learning of large language models. arXiv preprint arXiv:2411.06171","DOI":"10.18653\/v1\/2024.emnlp-main.190"},{"key":"11423_CR58","unstructured":"Hendrycks D, Burns C, Kadavath S, Arora A, Basart S, Tang E, Song D, Steinhardt, J (2021) Measuring mathematical problem solving with the math dataset. arXiv preprint arXiv:2103.03874"},{"key":"11423_CR59","unstructured":"Hinton G, Vinyals O, Dean J et al (2015) Distilling the knowledge in a neural network. arXiv preprint arXiv:1503.02531"},{"key":"11423_CR60","doi-asserted-by":"crossref","unstructured":"Hsieh C-Y, Li C-L, Yeh C-K, Nakhost H, Fujii Y, Ratner A, Krishna R, Lee C-Y, Pfister T (2023) Distilling step-by-step! outperforming larger language models with less training data and smaller model sizes. arXiv preprint arXiv:2305.02301","DOI":"10.18653\/v1\/2023.findings-acl.507"},{"key":"11423_CR61","unstructured":"Hsu D, Ji Z, Telgarsky M, Wang L (2021) Generalization bounds via distillation. arXiv preprint arXiv:2104.05641"},{"issue":"2","key":"11423_CR62","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3703155","volume":"43","author":"L Huang","year":"2025","unstructured":"Huang L, Yu W, Ma W, Zhong W, Feng Z, Wang H, Chen Q, Peng W, Feng X, Qin B et al (2025) A survey on hallucination in large language models: principles, taxonomy, challenges, and open questions. ACM Trans Inf Syst 43(2):1\u201355","journal-title":"ACM Trans Inf Syst"},{"key":"11423_CR63","unstructured":"Hu L, Huang T, Xie H, Gong X, Ren C, Hu Z, Yu L, Ma P, Wang D (2025) Semi-supervised concept bottleneck models. https:\/\/arxiv.org\/abs\/2406.18992"},{"key":"11423_CR64","doi-asserted-by":"crossref","unstructured":"Jayasuriya D, Tayebati S, Ettori D, Krishnan R, Trivedi AR (2025) Sparc: Subspace-aware prompt adaptation for robust continual learning in llms. arXiv preprint arXiv:2502.02909","DOI":"10.1109\/IJCNN64981.2025.11228226"},{"key":"11423_CR65","doi-asserted-by":"crossref","unstructured":"Jiao X, Yin Y, Shang L, Jiang X, Chen X, Li L, Wang F, Liu Q (2019) Tinybert: Distilling bert for natural language understanding. arXiv preprint arXiv:1909.10351","DOI":"10.18653\/v1\/2020.findings-emnlp.372"},{"key":"11423_CR66","doi-asserted-by":"publisher","first-page":"583","DOI":"10.1038\/s41586-021-03819-2","volume":"596","author":"J Jumper","year":"2021","unstructured":"Jumper J, Evans R, Pritzel A et al (2021) Highly accurate protein structure prediction with alphafold. Nature 596:583\u2013589. https:\/\/doi.org\/10.1038\/s41586-021-03819-2","journal-title":"Nature"},{"key":"11423_CR67","doi-asserted-by":"crossref","unstructured":"Kanagavelu R, Walia M, Wang Y, Fu H, Wei Q, Liu Y, Goh RSM (2024) Medsynth: Leveraging generative model for healthcare data sharing. In: International conference on medical image computing and computer-assisted intervention. Springer, pp 654\u2013664","DOI":"10.1007\/978-3-031-72390-2_61"},{"key":"11423_CR68","unstructured":"Kang I, Ram P, Zhou Y, Samulowitz H, Seneviratne O (2025) On learning representations for tabular data distillation. arXiv preprint arXiv:2501.13905"},{"key":"11423_CR69","doi-asserted-by":"crossref","unstructured":"Keswani M, Ramakrishnan S, Reddy N, Balasubramanian VN (2022) Proto2Proto: Can you recognize the car, the way I do?. https:\/\/arxiv.org\/abs\/2204.11830","DOI":"10.1109\/CVPR52688.2022.00999"},{"key":"11423_CR70","doi-asserted-by":"crossref","unstructured":"Khanuja S, Johnson M, Talukdar P (2021) Mergedistill: Merging pre-trained language models using distillation. arXiv preprint arXiv:2106.02834","DOI":"10.18653\/v1\/2021.findings-acl.254"},{"key":"11423_CR71","unstructured":"Khan A, Underwood R, Siebenschuh C, Babuji Y, Ajith A, Hippe K, Gokdemir O, Brace A, Chard K, Foster I (2024) Lshbloom: Memory-efficient, extreme-scale document deduplication. arXiv preprint arXiv:2411.04257"},{"key":"11423_CR72","unstructured":"Kim J-H, Kim J, Oh SJ, Yun S, Song H, Jeong J, Ha J-W, Song HO (2022) Dataset condensation via efficient synthetic-data parameterization. In: ICML"},{"issue":"13","key":"11423_CR73","doi-asserted-by":"publisher","first-page":"3521","DOI":"10.1073\/pnas.1611835114","volume":"114","author":"J Kirkpatrick","year":"2017","unstructured":"Kirkpatrick J, Pascanu R, Rabinowitz N, Veness J, Desjardins G, Rusu AA, Milan K, Quan J, Ramalho T, Grabska-Barwinska A et al (2017) Overcoming catastrophic forgetting in neural networks. Proc Natl Acad Sci USA 114(13):3521\u20133526","journal-title":"Proc Natl Acad Sci USA"},{"key":"11423_CR74","unstructured":"Kong X, Li L, Dou M, Chen Z, Wu Y, Guo G (2025) Quantum-enhanced llm efficient fine tuning. arXiv preprint arXiv:2503.12790"},{"key":"11423_CR75","unstructured":"Korattikara\u00a0Balan A, Rathod V, Murphy KP, Welling M (2015) Bayesian dark knowledge. Adv Neural Inf Process Syst 28"},{"key":"11423_CR76","doi-asserted-by":"crossref","unstructured":"Latif E, Fang L, Ma P, Zhai X (2024a) Knowledge distillation of llms for automatic scoring of science assessments. In: International conference on artificial intelligence in education. Springer, pp 166\u2013174","DOI":"10.1007\/978-3-031-64312-5_20"},{"key":"11423_CR77","unstructured":"Latif E, Zhou Y, Guo S, Gao Y, Shi L, Nayaaba M, Lee G, Zhang L, Bewersdorff A, Fang L et al (2024b) A systematic assessment of openai o1-preview for higher order thinking in education. arXiv preprint arXiv:2410.21287"},{"key":"11423_CR78","doi-asserted-by":"crossref","unstructured":"Lee G-G, Zhai X (2024) Using chatgpt for science learning: a study on pre-service teachers\u2019 lesson planning. IEEE Trans Learn Technol","DOI":"10.1109\/TLT.2024.3401457"},{"key":"11423_CR79","unstructured":"Lee HB, Lee DB, Hwang SJ (2022) Dataset condensation with latent space knowledge factorization and sharing. arXiv preprint arXiv:2208.10494"},{"key":"11423_CR80","doi-asserted-by":"crossref","unstructured":"Lee S, Zhou J, Ao C, Li K, Du X, He S, Liu J, Yang M, Wen Z, Ni S (2025) Distillation quantification for large language models. arXiv preprint arXiv:2501.12619","DOI":"10.18653\/v1\/2025.acl-long.248"},{"key":"11423_CR81","unstructured":"Li X (2025) A review of prominent paradigms for llm-based agents: tool use, planning (including rag), and feedback learning. In: Proceedings of the 31st international conference on computational linguistics, pp 9760\u20139779"},{"issue":"2","key":"11423_CR82","doi-asserted-by":"publisher","first-page":"1","DOI":"10.4018\/IJCPS.2020070101","volume":"2","author":"T Li","year":"2020","unstructured":"Li T, Meng C (2020) Modern subsampling methods for large-scale least squares regression. Int J Cyber-Phys Syst (IJCPS) 2(2):1\u201328","journal-title":"Int J Cyber-Phys Syst (IJCPS)"},{"key":"11423_CR83","first-page":"69490","volume":"36","author":"L Li","year":"2023","unstructured":"Li L, Dong P, Li A, Wei Z, Yang Y (2023a) Kd-zero: evolving knowledge distiller for any teacher-student pairs. Adv Neural Inf Process Syst 36:69490\u201369504","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR86","unstructured":"Li Y, Du M, Song R, Wang X, Wang Y (2023b) A survey on fairness in large language models. arXiv preprint arXiv:2308.10149"},{"key":"11423_CR90","unstructured":"Li X, Lin L, Wang S, Qian C (2023c) Unlock the power: competitive distillation for multi-modal large language models. arXiv preprint arXiv:2311.08213"},{"key":"11423_CR103","unstructured":"Li M, Yu J, Li T, Meng C (2023d) Core-elements for classical linear regression. arXiv preprint http:\/\/arxiv.org\/abs\/2206.102401398 rXiv:2206.102401398, pp 1399\u20131400"},{"key":"11423_CR105","doi-asserted-by":"crossref","unstructured":"Li Z, Zhu H, Lu Z, Yin M (2023e) Synthetic data generation with large language models for text classification: potential and limitations. arXiv preprint arXiv:2310.07849","DOI":"10.18653\/v1\/2023.emnlp-main.647"},{"issue":"6","key":"11423_CR84","doi-asserted-by":"publisher","first-page":"190","DOI":"10.1007\/s11222-024-10505-6","volume":"34","author":"M Li","year":"2024","unstructured":"Li M, Yu J, Li T, Meng C (2024a) Core-elements for large-scale least squares estimation. Stat Comput 34(6):190","journal-title":"Stat Comput"},{"key":"11423_CR85","unstructured":"Li J, Beeching E, Tunstall L, Lipkin B, Soletskyi R, Huang S, Rasul K et al (2024b) Numinamath: The largest public dataset in ai4maths with 860k pairs of competition math problems and solutions. Hugging Face Repository 13"},{"key":"11423_CR87","doi-asserted-by":"crossref","unstructured":"Li M, Guo Z, Wu Y, Guo P, Shi Y, Hu S, Wan W, Hu S (2024c) Vidta: Enhanced drug-target affinity prediction via virtual graph nodes and attention-based feature fusion. In: 2024 IEEE international conference on bioinformatics and biomedicine (BIBM). IEEE, pp 42\u201347","DOI":"10.1109\/BIBM62325.2024.10822490"},{"key":"11423_CR88","doi-asserted-by":"crossref","unstructured":"Li Y, Han SC, Dai Y, Cao F (2024d) Chulo: Chunk-level key information representation for long document processing. arXiv preprint arXiv:2410.11119","DOI":"10.18653\/v1\/2025.findings-acl.762"},{"key":"11423_CR89","unstructured":"Li X, He S, Wu J, Yang Z, Xu Y, Jun Y, Liu H, Liu K, Zhao J (2024e) Mode-cotd: Chain-of-thought distillation for complex reasoning tasks with mixture of decoupled lora-experts. In: Proceedings of the 2024 joint international conference on computational linguistics, language resources and evaluation (LREC-COLING 2024), pp 11475\u201311485"},{"key":"11423_CR91","doi-asserted-by":"crossref","unstructured":"Li L, Li G, Togo R, Maeda K, Ogawa T, Haseyama M (2024f) Generative dataset distillation: balancing global structure and local details. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 7664\u20137671","DOI":"10.1109\/CVPRW63382.2024.00762"},{"key":"11423_CR104","unstructured":"Li M, Zhou F, Song X (2024g) Bild: Bi-directional logits difference loss for large language model distillation. arXiv preprint arXiv:2406.13555"},{"key":"11423_CR93","doi-asserted-by":"crossref","unstructured":"Liu J, Cui L, Liu H, Huang D, Wang Y, Zhang Y (2020) Logiqa: A challenge dataset for machine reading comprehension with logical reasoning. arXiv preprint arXiv:2007.08124","DOI":"10.24963\/ijcai.2020\/501"},{"key":"11423_CR92","first-page":"1100","volume":"35","author":"S Liu","year":"2022","unstructured":"Liu S, Wang K, Yang X, Ye J, Wang X (2022) Dataset distillation via factorization. Adv Neural Inf Process Syst 35:1100\u20131113","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR94","doi-asserted-by":"crossref","unstructured":"Liu Y, Gu J, Wang K, Zhu Z, Jiang W, You Y (2023a) Dream: Efficient dataset distillation by representative matching. In: Proceedings of the IEEE\/CVF international conference on computer vision, pp 17314\u201317324","DOI":"10.1109\/ICCV51070.2023.01588"},{"key":"11423_CR100","unstructured":"Liu Y, Yao Y, Ton J-F, Zhang X, Guo R, Cheng H, Klochkov Y, Taufiq MF, Li H (2023b) Trustworthy llms: a survey and guideline for evaluating large language models\u2019 alignment. arXiv preprint arXiv:2308.05374"},{"key":"11423_CR101","unstructured":"Liu W, Zeng W, He K, Jiang Y, He J (2023c) What makes good data for alignment? A comprehensive study of automatic data selection in instruction tuning. In: The twelfth international conference on learning representations"},{"key":"11423_CR96","doi-asserted-by":"crossref","unstructured":"Liu Z, Liu L (2024) Graphsaid: Graph sampling via attention based integer programming method. In: Proceedings of the 23rd international conference on autonomous agents and multiagent systems, pp 1256\u20131264","DOI":"10.65109\/OBSH2603"},{"key":"11423_CR95","unstructured":"Liu C, Kang Y, Zhao F, Kuang K, Jiang Z, Sun C, Wu F (2024a) Evolving knowledge distillation with large language models and active learning. arXiv preprint arXiv:2403.06414"},{"key":"11423_CR97","unstructured":"Liu Z, Liu Q, Li Y, Liu L, Shrivastava A, Bi S, Hong L, Chi EH, Zhao Z (2024b) Wisdom of committee: distilling from foundation model to specialized application model. arXiv preprint arXiv:2402.14035"},{"key":"11423_CR99","doi-asserted-by":"crossref","unstructured":"Liu Y, Wu Z, Lu Z, Nie C, Wen G, Hu P, Zhu X (2024c) Noisy node classification by bi-level optimization based multi-teacher distillation. arXiv preprint arXiv:2404.17875","DOI":"10.1609\/aaai.v39i18.34095"},{"key":"11423_CR102","unstructured":"Liu S, Zhang B, Huang Z (2024d) Benchmark real-time adaptation and communication capabilities of embodied agent in collaborative scenarios. arXiv preprint arXiv:2412.00435"},{"key":"11423_CR98","unstructured":"Liu B, Li X, Zhang J, Wang J, He T, Hong S, Liu H, Zhang S, Song K, Zhu K et al (2025) Advances and challenges in foundation agents: from brain-inspired intelligence to evolutionary, collaborative, and safe systems. arXiv preprint arXiv:2504.01990"},{"key":"11423_CR106","first-page":"13877","volume":"35","author":"N Loo","year":"2022","unstructured":"Loo N, Hasani R, Amini A, Rus D (2022) Efficient dataset distillation using random feature approximation. Adv Neural Inf Process Syst 35:13877\u201313891","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR107","unstructured":"Lopez-Paz D, Bottou L, Sch\u00f6lkopf B, Vapnik V (2015) Unifying distillation and privileged information. arXiv preprint arXiv:1511.03643"},{"issue":"3","key":"11423_CR108","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1093\/bib\/bbad145","volume":"24","author":"R Lu","year":"2023","unstructured":"Lu R, Wang J, Li P, Li Y, Tan S, Pan Y, Liu H, Gao P, Xie G, Yao X (2023) Improving drug-target affinity prediction via feature fusion and knowledge distillation. Brief Bioinform 24(3):145","journal-title":"Brief Bioinform"},{"issue":"1","key":"11423_CR109","first-page":"861","volume":"16","author":"P Ma","year":"2015","unstructured":"Ma P, Mahoney MW, Yu B (2015) A statistical perspective on algorithmic leveraging. J Mach Learn Res 16(1):861\u2013911","journal-title":"J Mach Learn Res"},{"issue":"177","key":"11423_CR110","first-page":"1","volume":"23","author":"P Ma","year":"2022","unstructured":"Ma P, Chen Y, Zhang X, Xing X, Ma J, Mahoney MW (2022) Asymptotic analysis of sampling estimators for randomized numerical linear algebra algorithms. J Mach Learn Res 23(177):1\u201345","journal-title":"J Mach Learn Res"},{"issue":"3","key":"11423_CR111","doi-asserted-by":"publisher","first-page":"448","DOI":"10.1162\/neco.1992.4.3.448","volume":"4","author":"DJ MacKay","year":"1992","unstructured":"MacKay DJ (1992) A practical Bayesian framework for backpropagation networks. Neural Comput 4(3):448\u2013472","journal-title":"Neural Comput"},{"key":"11423_CR112","doi-asserted-by":"crossref","unstructured":"Maekawa A, Kosugi S, Funakoshi K, Okumura M (2024) Dilm: distilling dataset into language model for text-level dataset distillation. arXiv preprint arXiv:2404.00264","DOI":"10.18653\/v1\/2024.findings-naacl.199"},{"key":"11423_CR113","unstructured":"Malinin A, Mlodozeniec B, Gales M (2019) Ensemble distribution distillation. arXiv preprint arXiv:1905.00076"},{"key":"11423_CR114","unstructured":"Marion M, \u00dcst\u00fcn A, Pozzobon L, Wang A, Fadaee M, Hooker S (2023) When less is more: investigating data pruning for pretraining llms at scale. arXiv preprint arXiv:2309.04564"},{"issue":"5","key":"11423_CR115","doi-asserted-by":"publisher","first-page":"255","DOI":"10.3390\/info14050255","volume":"14","author":"T Mehmood","year":"2023","unstructured":"Mehmood T, Gerevini AE, Lavelli A, Olivato M, Serina I (2023) Distilling knowledge with a teacher\u2019s multitask model for biomedical named entity recognition. Information 14(5):255","journal-title":"Information"},{"key":"11423_CR116","doi-asserted-by":"publisher","first-page":"280","DOI":"10.4018\/978-1-5225-2498-4.ch014","volume-title":"Handbook of research on applied cybernetics and systems science","author":"C Meng","year":"2017","unstructured":"Meng C, Wang Y, Zhang X, Mandal A, Zhong W, Ma P (2017) Effective statistical methods for big data analytics. Handbook of research on applied cybernetics and systems science. IGI Global Scientific Publishing, Hershey, pp 280\u2013299"},{"issue":"3","key":"11423_CR117","doi-asserted-by":"publisher","first-page":"723","DOI":"10.1093\/biomet\/asaa019","volume":"107","author":"C Meng","year":"2020","unstructured":"Meng C, Zhang X, Zhang J, Zhong W, Ma P (2020) More efficient approximation of smoothing splines via space-filling basis selection. Biometrika 107(3):723\u2013735","journal-title":"Biometrika"},{"key":"11423_CR118","unstructured":"Menon AK, Rawat AS, Reddi S, Kim S, Kumar S (2021) A statistical perspective on distillation. In: International conference on machine learning. PMLR, pp 7632\u20137642"},{"key":"11423_CR119","unstructured":"Michel P, Levy O, Neubig G (2019) Are sixteen heads really better than one? Adv Neural Inf Process Syst 32"},{"key":"11423_CR120","doi-asserted-by":"crossref","unstructured":"Min S, Krishna K, Lyu X, Lewis M, Yih W-t, Koh PW, Iyyer M, Zettlemoyer L, Hajishirzi H (2023) Factscore: Fine-grained atomic evaluation of factual precision in long form text generation. arXiv preprint arXiv:2305.14251","DOI":"10.18653\/v1\/2023.emnlp-main.741"},{"key":"11423_CR121","first-page":"5191","volume":"34","author":"SI Mirzadeh","year":"2020","unstructured":"Mirzadeh SI, Farajtabar M, Li A, Levine N, Matsukawa A, Ghasemzadeh H (2020) Improved knowledge distillation via teacher assistant. Proc AAAI Conf Artif Intell 34:5191\u20135198","journal-title":"Proc AAAI Conf Artif Intell"},{"key":"11423_CR122","first-page":"11465","volume":"33","author":"B Mirzasoleiman","year":"2020","unstructured":"Mirzasoleiman B, Cao K, Leskovec J (2020) Coresets for robust training of deep neural networks against noisy labels. Adv Neural Inf Process Syst 33:11465\u201311477","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR123","first-page":"3351","volume":"33","author":"H Mobahi","year":"2020","unstructured":"Mobahi H, Farajtabar M, Bartlett P (2020) Self-distillation amplifies regularization in Hilbert space. Adv Neural Inf Process Syst 33:3351\u20133361","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR124","doi-asserted-by":"crossref","unstructured":"Moon T, Li L, Chu W, Liao C, Zheng Z, Chang Y (2010) Online learning for recency search ranking using real-time user feedback. In: Proceedings of the 19th ACM international conference on information and knowledge management, pp 1501\u20131504","DOI":"10.1145\/1871437.1871657"},{"key":"11423_CR125","unstructured":"Moore RC, Lewis W (2010) Intelligent selection of language model training data. In: Proceedings of the ACL 2010 conference short papers, pp 220\u2013224"},{"key":"11423_CR126","unstructured":"M\u00fcller R, Kornblith S, Hinton GE (2019) When does label smoothing help? Adv Neural Inf Process Syst 32"},{"key":"11423_CR127","doi-asserted-by":"publisher","unstructured":"Nawrot P (2023) nanoT5: Fast & simple pre-training and fine-tuning of T5 models with limited resources. In: Proceedings of the 3rd workshop for natural language processing open source software (NLP-OSS 2023). Association for Computational Linguistics, Singapore. https:\/\/doi.org\/10.18653\/v1\/2023.nlposs-1.11 . https:\/\/aclanthology.org\/2023.nlposs-1.11","DOI":"10.18653\/v1\/2023.nlposs-1.11"},{"issue":"9","key":"11423_CR128","doi-asserted-by":"publisher","first-page":"1953","DOI":"10.1093\/jamia\/ocae073","volume":"31","author":"M Nievas","year":"2024","unstructured":"Nievas M, Basu A, Wang Y, Singh H (2024) Distilling large language models for matching patients to clinical trials. J Am Med Inform Assoc 31(9):1953\u20131963","journal-title":"J Am Med Inform Assoc"},{"key":"11423_CR129","first-page":"21933","volume":"35","author":"Y Niu","year":"2022","unstructured":"Niu Y, Chen L, Zhou C, Zhang H (2022) Respecting transfer gap in knowledge distillation. Adv Neural Inf Process Syst 35:21933\u201321947","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR130","unstructured":"Niu S, Ma J, Bai L, Wang Z, Xu Y, Song Y, Yang X (2024) Multimodal clinical reasoning through knowledge-augmented rationale generation. arXiv preprint arXiv:2411.07611"},{"key":"11423_CR131","first-page":"27730","volume":"35","author":"L Ouyang","year":"2022","unstructured":"Ouyang L, Wu J, Jiang X, Almeida D, Wainwright C, Mishkin P, Zhang C, Agarwal S, Slama K, Ray A et al (2022) Training language models to follow instructions with human feedback. Adv Neural Inf Process Syst 35:27730\u201327744","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR132","doi-asserted-by":"crossref","unstructured":"Park W, Kim D, Lu Y, Cho M (2019) Relational knowledge distillation. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 3967\u20133976","DOI":"10.1109\/CVPR.2019.00409"},{"key":"11423_CR133","first-page":"79155","volume":"36","author":"G Penedo","year":"2023","unstructured":"Penedo G, Malartic Q, Hesslow D, Cojocaru R, Alobeidli H, Cappelli A, Pannier B, Almazrouei E, Launay J (2023) The refined web dataset for falcon llm: outperforming curated corpora with web data only. Adv Neural Inf Process Syst 36:79155\u201379172","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR134","unstructured":"Phuong M, Lampert C (2019) Towards understanding knowledge distillation. In: International conference on machine learning. PMLR, pp 5142\u20135151"},{"key":"11423_CR135","first-page":"4816","volume":"34","author":"K Pillutla","year":"2021","unstructured":"Pillutla K, Swayamdipta S, Zellers R, Thickstun J, Welleck S, Choi Y, Harchaoui Z (2021) Mauve: measuring the gap between neural text and human text using divergence frontiers. Adv Neural Inf Process Syst 34:4816\u20134828","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR136","doi-asserted-by":"publisher","first-page":"124990","DOI":"10.1109\/ACCESS.2022.3224788","volume":"10","author":"SN Prabhakar","year":"2022","unstructured":"Prabhakar SN, Deshwal A, Mishra R, Kim H (2022) Distilnas: neural architecture search with distilled data. IEEE Access 10:124990\u2013124998","journal-title":"IEEE Access"},{"key":"11423_CR137","doi-asserted-by":"crossref","unstructured":"Pryzant R, Iter D, Li J, Lee YT, Zhu C, Zeng M (2023) Automatic prompt optimization with \"gradient descent\" and beam search. arXiv preprint arXiv:2305.03495","DOI":"10.18653\/v1\/2023.emnlp-main.494"},{"key":"11423_CR138","unstructured":"Qin Z, Dong Q, Zhang X, Dong L, Huang X, Yang Z, Khademi M, Zhang D, Awadalla HH, Fung YR et al (2025) Scaling laws of synthetic data for language models. arXiv preprint arXiv:2503.19551"},{"key":"11423_CR140","unstructured":"Qu Z, Yin L, Yu Z, Wang W et al (2024) Coursegpt-zh: An educational large language model based on knowledge distillation incorporating prompt optimization. arXiv preprint arXiv:2405.04781"},{"key":"11423_CR139","doi-asserted-by":"crossref","unstructured":"Qu G, Chen Q, Wei W, Lin Z, Chen X, Huang K (2025) Mobile edge intelligence for large language models: a contemporary survey. IEEE Commun Surv Tutor","DOI":"10.36227\/techrxiv.172115025.57884352\/v1"},{"key":"11423_CR141","unstructured":"Rae JW, Borgeaud S, Cai T, Millican K, Hoffmann J, Song F, Aslanides J, Henderson S, Ring R, Young S et al (2021) Scaling language models: methods, analysis & insights from training gopher. arXiv preprint arXiv:2112.11446"},{"key":"11423_CR142","doi-asserted-by":"publisher","first-page":"26839","DOI":"10.1109\/ACCESS.2024.3365742","volume":"12","author":"MAK Raiaan","year":"2024","unstructured":"Raiaan MAK, Mukta MSH, Fatema K, Fahad NM, Sakib S, Mim MMJ, Ahmad J, Ali ME, Azam S (2024) A review on large language models: architectures, applications, taxonomies, open issues and challenges. IEEE Access 12:26839\u201326874","journal-title":"IEEE Access"},{"key":"11423_CR143","doi-asserted-by":"publisher","unstructured":"Ramamurthy P, Aakur SN (2022) Isd-qa: Iterative distillation of commonsense knowledge from general language models for unsupervised question answering. In: 2022 26th International conference on pattern recognition (ICPR), pp 1229\u20131235. https:\/\/doi.org\/10.1109\/ICPR56361.2022.9956441","DOI":"10.1109\/ICPR56361.2022.9956441"},{"key":"11423_CR144","unstructured":"Rejeleene R, Xu X, Talburt J (2024) Towards trustable language models: investigating information quality of large language models. arXiv preprint arXiv:2401.13086"},{"key":"11423_CR145","unstructured":"Romero A, Ballas N, Kahou SE, Chassang A, Gatta C, Bengio Y (2014) Fitnets: Hints for thin deep nets. arXiv preprint arXiv:1412.6550"},{"key":"11423_CR146","unstructured":"Sachdeva N, McAuley J (2023) Data distillation: a survey. Trans Mach Learn Res"},{"key":"11423_CR147","doi-asserted-by":"crossref","unstructured":"Sajedi A, Khaki S, Liu LZ, Amjadian E, Lawryshyn YA, Plataniotis KN (2024) Data-to-model distillation: data-efficient learning framework. In: European conference on computer vision. Springer, pp 438\u2013457","DOI":"10.1007\/978-3-031-72775-7_25"},{"issue":"1","key":"11423_CR148","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1007\/s10791-025-09657-4","volume":"28","author":"H Salman","year":"2025","unstructured":"Salman H, Zaki C, Charara N, Guehis S, Pradat-Peyre J-F, Nasser A (2025) Knowledge distillation in federated learning: a comprehensive survey. Discover Comput 28(1):145","journal-title":"Discover Comput"},{"key":"11423_CR149","unstructured":"Sanh V, Debut L, Chaumond J, Wolf T (2019) Distilbert, a distilled version of bert: smaller, faster, cheaper and lighter. arXiv preprint arXiv:1910.01108"},{"key":"11423_CR150","volume-title":"Should robots replace teachers?: AI and the future of education","author":"N Selwyn","year":"2019","unstructured":"Selwyn N (2019) Should robots replace teachers?: AI and the future of education. Wiley, Hoboken"},{"key":"11423_CR151","doi-asserted-by":"crossref","unstructured":"Shang J, Peng C, Ji Y, Guan J, Cai D, Tang X, Sun Y (2024) Accurate and efficient protein embedding using multi-teacher distillation learning. arXiv preprint arXiv:2405.11735","DOI":"10.1093\/bioinformatics\/btae567"},{"key":"11423_CR152","doi-asserted-by":"crossref","unstructured":"Shao S, Yin Z, Zhou M, Zhang X, Shen Z (2024) Generalized large-scale data condensation via various backbone and statistical matching. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 16709\u201316718","DOI":"10.1109\/CVPR52733.2024.01581"},{"key":"11423_CR153","unstructured":"Shi H, Gao J, Ren X, Xu H, Liang X, Li Z, Kwok JT (2021) SparseBERT: rethinking the importance analysis in self-attention. arXiv:2102.12871"},{"key":"11423_CR154","first-page":"70033","volume":"36","author":"D Shin","year":"2023","unstructured":"Shin D, Shin S, Moon I-C (2023) Frequency domain-based dataset distillation. Adv Neural Inf Process Syst 36:70033\u201370044","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR155","doi-asserted-by":"crossref","unstructured":"Shokri R, Stronati M, Song C, Shmatikov V (2017) Membership inference attacks against machine learning models. In: 2017 IEEE symposium on security and privacy (SP). IEEE, pp 3\u201318","DOI":"10.1109\/SP.2017.41"},{"key":"11423_CR156","first-page":"7059","volume":"2023","author":"K Shridhar","year":"2023","unstructured":"Shridhar K, Stolfo A, Sachan M (2023) Distilling reasoning capabilities into smaller language models. Find Assoc Comput Linguist ACL 2023:7059\u20137073","journal-title":"Find Assoc Comput Linguist ACL"},{"key":"11423_CR157","doi-asserted-by":"crossref","unstructured":"Shum K, Xu M, Zhang J, Chen Z, Diao S, Dong H, Zhang J, Raza MO (2024) First: Teach a reliable large language model through efficient trustworthy distillation. arXiv preprint arXiv:2408.12168","DOI":"10.18653\/v1\/2024.emnlp-main.703"},{"key":"11423_CR158","doi-asserted-by":"crossref","unstructured":"Song R, Liu D, Chen DZ, Festag A, Trinitis C, Schulz M, Knoll A (2023) Federated learning via decentralized dataset distillation in resource-constrained edge environments. In: 2023 International joint conference on neural networks (IJCNN). IEEE, pp 1\u201310","DOI":"10.1109\/IJCNN54540.2023.10191879"},{"key":"11423_CR159","first-page":"19523","volume":"35","author":"B Sorscher","year":"2022","unstructured":"Sorscher B, Geirhos R, Shekhar S, Ganguli S, Morcos A (2022) Beyond neural scaling laws: beating power law scaling via data pruning. Adv Neural Inf Process Syst 35:19523\u201319536","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR160","first-page":"6906","volume":"34","author":"S Stanton","year":"2021","unstructured":"Stanton S, Izmailov P, Kirichenko P, Alemi AA, Wilson AG (2021) Does knowledge distillation really work? Adv Neural Inf Process Syst 34:6906\u20136919","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR161","doi-asserted-by":"crossref","unstructured":"Sucholutsky I, Schonlau M (2021) Soft-label dataset distillation and text dataset distillation. In: 2021 International joint conference on neural networks (IJCNN). IEEE, pp 1\u20138","DOI":"10.1109\/IJCNN52387.2021.9533769"},{"key":"11423_CR163","doi-asserted-by":"crossref","unstructured":"Sun S, Cheng Y, Gan Z, Liu J (2019) Patient knowledge distillation for Bert model compression. arXiv preprint arXiv:1908.09355","DOI":"10.18653\/v1\/D19-1441"},{"issue":"1","key":"11423_CR162","doi-asserted-by":"publisher","first-page":"149","DOI":"10.1093\/biomet\/asaa047","volume":"108","author":"X Sun","year":"2020","unstructured":"Sun X, Zhong W, Ma P (2020) An asymptotic and empirical smoothing parameters selection method for smoothing spline ANOVA models in large samples. Biometrika 108(1):149\u2013166. https:\/\/doi.org\/10.1093\/biomet\/asaa047","journal-title":"Biometrika"},{"key":"11423_CR164","unstructured":"Sun L, Gou J, Yu B, Du L, Tao D (2021) Collaborative teacher-student learning via multiple knowledge transfer. arXiv preprint arXiv:2101.08471"},{"key":"11423_CR165","doi-asserted-by":"crossref","unstructured":"Sun P, Shi B, Yu D, Lin T (2024) On the diversity and realism of distilled dataset: an efficient dataset distillation paradigm. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 9390\u20139399","DOI":"10.1109\/CVPR52733.2024.00897"},{"key":"11423_CR166","doi-asserted-by":"crossref","unstructured":"Sutanto P, Santoso J, Setiawan EI, Wibawa AP (2024) Llm distillation for efficient few-shot multiple choice question answering. arXiv preprint arXiv:2403.00633","DOI":"10.18653\/v1\/2025.findings-emnlp.452"},{"key":"11423_CR167","unstructured":"Tan H, Wu S, Huang W, Zhao S, Qi X (2025) Data pruning by information maximization. In: The thirteenth international conference on learning representations"},{"issue":"6","key":"11423_CR168","first-page":"7","volume":"3","author":"R Taori","year":"2023","unstructured":"Taori R, Gulrajani I, Zhang T, Dubois Y, Li X, Guestrin C, Liang P, Hashimoto TB (2023a) Alpaca: a strong, replicable instruction-following model. Stanford Center Res Found Models 3(6):7 (https:\/\/crfmstanford.edu\/2023\/03\/13\/alpaca.html)","journal-title":"Stanford Center Res Found Models"},{"key":"11423_CR169","volume-title":"Stanford alpaca: an instruction-following llama model","author":"R Taori","year":"2023","unstructured":"Taori R, Gulrajani I, Zhang T, Dubois Y, Li X, Guestrin C, Liang P, Hashimoto TB (2023b) Stanford alpaca: an instruction-following llama model. Stanford, CA, USA"},{"key":"11423_CR170","doi-asserted-by":"crossref","unstructured":"Tariq A, Purkayastha S, Dai T (2024) Patient centric summarization of radiology findings using large language models. arXiv preprint arXiv:2402.01562","DOI":"10.1101\/2024.02.01.24302145"},{"issue":"6","key":"11423_CR171","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3530811","volume":"55","author":"Y Tay","year":"2022","unstructured":"Tay Y, Dehghani M, Bahri D, Metzler D (2022) Efficient transformers: a survey. ACM Comput Surv 55(6):1\u201328","journal-title":"ACM Comput Surv"},{"issue":"5","key":"11423_CR172","doi-asserted-by":"publisher","first-page":"0000503","DOI":"10.1371\/journal.pdig.0000503","volume":"3","author":"T Templin","year":"2024","unstructured":"Templin T, Perez MW, Sylvia S, Leek J, Sinnott-Armstrong N (2024) Addressing 6 challenges in generative ai for digital health: a scoping review. PLOS Digital Health 3(5):0000503","journal-title":"PLOS Digital Health"},{"key":"11423_CR173","doi-asserted-by":"crossref","unstructured":"Tian Y, Han Y, Chen X, Wang W, Chawla NV (2024) Beyond answers: transferring reasoning capabilities to smaller llms using multi-teacher knowledge distillation. arXiv preprint arXiv:2402.04616","DOI":"10.1145\/3701551.3703577"},{"key":"11423_CR174","doi-asserted-by":"crossref","unstructured":"Tian K, Mitchell E, Zhou A, Sharma A, Rafailov R, Yao H, Finn C, Manning CD (2023) Just ask for calibration: strategies for eliciting calibrated confidence scores from language models fine-tuned with human feedback. arXiv preprint arXiv:2305.14975","DOI":"10.18653\/v1\/2023.emnlp-main.330"},{"key":"11423_CR175","unstructured":"Touvron H, Lavril T, Izacard G, Martinet X, Lachaux M-A, Lacroix T, Rozi\u00e8re B, Goyal N, Hambro E, Azhar F et al (2023) Llama: Open and efficient foundation language models. arXiv preprint arXiv:2302.13971"},{"issue":"1","key":"11423_CR176","doi-asserted-by":"publisher","first-page":"041","DOI":"10.1093\/biomethods\/bpaf041","volume":"10","author":"T-X Tran","year":"2025","unstructured":"Tran T-X, Nguyen T-T, Le N-Q-K, Nguyen V-N (2025) Kd_multisucc: incorporating multi-teacher knowledge distillation and word embeddings for cross-species prediction of protein succinylation sites. Biol Methods Protocols 10(1):041","journal-title":"Biol Methods Protocols"},{"key":"11423_CR177","unstructured":"Vadera M, Jalaian B, Marlin B (2020) Generalized Bayesian posterior expectation distillation for deep neural networks. In: Conference on uncertainty in artificial intelligence. PMLR, pp 719\u2013728"},{"issue":"1","key":"11423_CR178","first-page":"2023","volume":"16","author":"V Vapnik","year":"2015","unstructured":"Vapnik V, Izmailov R et al (2015) Learning using privileged information: similarity control and knowledge transfer. J Mach Learn Res 16(1):2023\u20132049","journal-title":"J Mach Learn Res"},{"key":"11423_CR179","unstructured":"Vedula KS, Gupta A, Swaminathan A et al (2024) Distilling large language models for efficient clinical information extraction. arXiv preprint arXiv:2403.04138"},{"key":"11423_CR180","doi-asserted-by":"crossref","unstructured":"Wadhwa S, Shaib C, Amir S, Wallace BC (2025) Who taught you that? Tracing teachers in model distillation. arXiv preprint arXiv:2502.06659","DOI":"10.18653\/v1\/2025.findings-acl.173"},{"key":"11423_CR192","unstructured":"Wang T, Zhu J-Y, Torralba A, Efros AA (2018) Dataset distillation. arXiv preprint arXiv:1811.10959"},{"key":"11423_CR190","unstructured":"Wang B, Xu C, Wang S, Gan Z, Cheng Y, Gao J, Awadallah AH, Li B (2021) Adversarial glue: a multi-task benchmark for robustness evaluation of language models. arXiv preprint arXiv:2111.02840"},{"key":"11423_CR184","first-page":"4620","volume":"36","author":"Q Wang","year":"2022","unstructured":"Wang Q, Chen J, Zhou Y, Li Y, Zheng L, Wang S, Li Z, Cui S (2022a) Contact-distil: boosting low homologous protein contact map prediction by self-supervised distillation. Proc AAAI Conf Artif Intell 36:4620\u20134627","journal-title":"Proc AAAI Conf Artif Intell"},{"key":"11423_CR188","doi-asserted-by":"crossref","unstructured":"Wang Y, Kordi Y, Mishra S, Liu A, Smith NA, Khashabi D, Hajishirzi H (2022b) Self-instruct: aligning language models with self-generated instructions. arXiv preprint arXiv:2212.10560","DOI":"10.18653\/v1\/2023.acl-long.754"},{"key":"11423_CR189","doi-asserted-by":"crossref","unstructured":"Wang Y, Xu C, Sun Q, Hu H, Tao C, Geng X, Jiang D (2022c) Promda: Prompt-based data augmentation for low-resource nlu tasks. arXiv preprint arXiv:2202.12499","DOI":"10.18653\/v1\/2022.acl-long.292"},{"key":"11423_CR187","unstructured":"Wang J, Hu X, Hou W, Chen H, Zheng R, Wang Y, Yang L, Huang H, Ye W, Geng X et al (2023) On the robustness of chatgpt: an adversarial and out-of-distribution perspective. arXiv preprint arXiv:2302.12095"},{"key":"11423_CR183","unstructured":"Wang JT, Yang T, Zou J, Kwon Y, Jia R (2024a) Rethinking data Shapley for data selection tasks: misleads and merits. In: International conference on machine learning"},{"key":"11423_CR185","unstructured":"Wang W, Chen W, Luo Y, Long Y, Lin Z, Zhang L, Lin B, Cai D, He X (2024b) Model compression and efficient inference for large language models: a survey. arXiv preprint arXiv:2402.09748"},{"key":"11423_CR186","doi-asserted-by":"crossref","unstructured":"Wang Y, Chen Z, Yang D, Sun Y, Qi L (2024c) Self-cooperation knowledge distillation for novel class discovery. arXiv:2407.01930","DOI":"10.1007\/978-3-031-72897-6_26"},{"key":"11423_CR191","doi-asserted-by":"crossref","unstructured":"Wang F, Zhang Z, Zhang X, Wu Z, Mo T, Lu Q, Wang W, Li R, Xu J, Tang X et al (2024d) A comprehensive survey of small language models in the era of large language models: techniques, enhancements, applications, collaboration with llms, and trustworthiness. arXiv preprint arXiv:2411.03350","DOI":"10.1145\/3768165"},{"key":"11423_CR181","unstructured":"Wang JT, Mittal P, Song D, Jia R (2025a) Data Shapley in one training run. In: International conference on learning representations"},{"key":"11423_CR182","unstructured":"Wang JT, Song D, Zou J, Mittal P, Jia R (2025b) Capturing the temporal dependence of training data influence. In: International conference on learning representations"},{"key":"11423_CR193","unstructured":"Wei J, Bosma M, Zhao VY, Guu K, Yu AW, Lester B, Du N, Dai AM, Le QV (2021) Finetuned language models are zero-shot learners. arXiv preprint arXiv:2109.01652"},{"key":"11423_CR194","unstructured":"Wei J, Tay Y, Bommasani R, Raffel C, Zoph B, Borgeaud S, Yogatama D, Bosma M, Zhou D, Metzler D et al (2022) Emergent abilities of large language models. arXiv preprint arXiv:2206.07682"},{"key":"11423_CR195","unstructured":"Welling M, Teh YW (2011) Bayesian learning via stochastic gradient Langevin dynamics. In: Proceedings of the 28th international conference on machine learning (ICML-11), pp 681\u2013688"},{"key":"11423_CR196","unstructured":"Wenzek G, Lachaux M-A, Conneau A, Chaudhary V, Guzm\u00e1n F, Joulin A, Grave \u00c9 (2020) Ccnet: Extracting high quality monolingual datasets from web crawl data. In: Proceedings of the twelfth language resources and evaluation conference, pp 4003\u20134012"},{"key":"11423_CR197","doi-asserted-by":"crossref","unstructured":"Wilkins J, Rodriguez M (2024) Higher performance of mistral large on mmlu benchmark through two-stage knowledge distillation","DOI":"10.21203\/rs.3.rs-4410506\/v1"},{"key":"11423_CR198","unstructured":"Wu S, Cheng H, Cai J, Ma P, Zhong W (2023a) Subsampling in large graphs using ricci curvature. In: International conference on learning representations"},{"key":"11423_CR200","doi-asserted-by":"crossref","unstructured":"Wu M, Waheed A, Zhang C, Abdul-Mageed M, Aji AF (2023b) Lamini-lm: A diverse herd of distilled models from large-scale instructions. arXiv preprint arXiv:2304.14402","DOI":"10.18653\/v1\/2024.eacl-long.57"},{"key":"11423_CR199","doi-asserted-by":"crossref","unstructured":"Wu F, Li Z, Li Y, Ding B, Gao J (2024) Fedbiot: Llm local fine-tuning in federated learning without full model. In: Proceedings of the 30th ACM SIGKDD conference on knowledge discovery and data mining, pp 3345\u20133355","DOI":"10.1145\/3637528.3671897"},{"key":"11423_CR201","doi-asserted-by":"crossref","unstructured":"Wu O, Yao R (2025) Data optimization in deep learning: a survey. IEEE Trans Know Data Eng","DOI":"10.1049\/icp.2025.3099"},{"key":"11423_CR202","unstructured":"Xia M, Malladi S, Gururangan S, Arora S, Chen D (2024) Less: selecting influential data for targeted instruction tuning. In: International conference on machine learning. PMLR, pp 54104\u201354132"},{"key":"11423_CR203","unstructured":"Xiao L, He Y (2024) Are large-scale soft labels necessary for large-scale dataset distillation? arXiv preprint arXiv:2410.15919"},{"key":"11423_CR204","unstructured":"Xie T, Wan Y, Huang W, Yin Z, Liu Y, Wang S, Linghu Q, Kit C, Grazian C, Zhang W et al (2023) Darwin series: domain specific large language models for natural science. arXiv preprint arXiv:2308.13565"},{"key":"11423_CR205","doi-asserted-by":"crossref","unstructured":"Xie T, Wan Y, Liu Y, Zeng Y, Wang S, Zhang W, Grazian C, Kit C, Ouyang W, Zhou D et al (2024) Darwin 1.5: Large language models as materials science adapted learners. arXiv preprint arXiv:2412.11970","DOI":"10.21203\/rs.3.rs-6752901\/v1"},{"key":"11423_CR206","unstructured":"Xu X, Li M, Tao C, Shen T, Cheng R, Li J, Xu C, Tao D, Zhou T (2024) A survey on knowledge distillation of large language models. arXiv preprint arXiv:2402.13116"},{"key":"11423_CR207","unstructured":"Yagnik N, Jhaveri J, Sharma V, Pila G (2024) Medlm: Exploring language models for medical question answering. arXiv preprint arXiv:2401.11389"},{"key":"11423_CR208","doi-asserted-by":"crossref","unstructured":"Yan K (2024) Optimizing an english text reading recommendation model by integrating collaborative filtering algorithm and fasttext classification method. Heliyon 10(9)","DOI":"10.1016\/j.heliyon.2024.e30413"},{"key":"11423_CR212","doi-asserted-by":"crossref","unstructured":"Yang Z, Qi P, Zhang S, Bengio Y, Cohen WW, Salakhutdinov R, Manning CD (2018) Hotpotqa: A dataset for diverse, explainable multi-hop question answering. In: Proceedings of the 2018 conference on empirical methods in natural language processing, pp 2369\u20132380","DOI":"10.18653\/v1\/D18-1259"},{"key":"11423_CR213","unstructured":"Yang S, Xie Z, Peng H, Xu M, Sun M, Li P (2022) Dataset pruning: reducing training data by examining generalization influence. arXiv preprint arXiv:2205.09329"},{"key":"11423_CR209","doi-asserted-by":"publisher","first-page":"115","DOI":"10.1038\/s41392-023-01381-z","volume":"8","author":"Z Yang","year":"2023","unstructured":"Yang Z, Zeng X, Zhao Y et al (2023a) Alphafold2 and its applications in the fields of biology and medicine. Sig Transduct Target Ther 8:115. https:\/\/doi.org\/10.1038\/s41392-023-01381-z","journal-title":"Sig Transduct Target Ther"},{"key":"11423_CR211","unstructured":"Yang W, Lin Y, Zhou J, Wen J (2023b) Enabling large language models to learn from rules. arXiv preprint arXiv:2311.08883"},{"key":"11423_CR210","unstructured":"Yang K, Klein D, Celikyilmaz A, Peng N, Tian Y (2024) Rlcd: Reinforcement learning from contrastive distillation for lm alignment. In: The twelfth international conference on learning representations"},{"key":"11423_CR214","doi-asserted-by":"crossref","unstructured":"Yao T, Zhai Z, Gao B (2020) Text classification model based on fasttext. In: 2020 IEEE international conference on artificial intelligence and information systems (ICAIIS). IEEE, pp 154\u2013157","DOI":"10.1109\/ICAIIS49377.2020.9194939"},{"key":"11423_CR216","doi-asserted-by":"crossref","unstructured":"Yin H, Molchanov P, Alvarez JM, Li Z, Mallya A, Hoiem D, Jha NK, Kautz J (2020) Dreaming to distill: data-free knowledge transfer via deepinversion. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 8715\u20138724","DOI":"10.1109\/CVPR42600.2020.00874"},{"key":"11423_CR215","first-page":"73582","volume":"36","author":"Z Yin","year":"2023","unstructured":"Yin Z, Xing E, Shen Z (2023) Squeeze, recover and relabel: dataset condensation at imagenet scale from a new perspective. Adv Neural Inf Process Syst 36:73582\u201373603","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR217","unstructured":"Yin Z, Shen Z (2023) Dataset distillation in large data era. arXiv preprint arXiv:2311.18838"},{"key":"11423_CR218","unstructured":"Yin Z, Shen Z (2024) Dataset distillation via curriculum data synthesis in large data era. Trans Mach Learn Res"},{"key":"11423_CR219","doi-asserted-by":"crossref","unstructured":"You S, Xu C, Xu C, Tao D (2017) Learning from multiple teacher networks. In: Proceedings of the 23rd ACM SIGKDD international conference on knowledge discovery and data mining, pp 1285\u20131294","DOI":"10.1145\/3097983.3098135"},{"issue":"1","key":"11423_CR220","doi-asserted-by":"publisher","first-page":"150","DOI":"10.1109\/TPAMI.2023.3323376","volume":"46","author":"R Yu","year":"2023","unstructured":"Yu R, Liu S, Wang X (2023) Dataset distillation: a comprehensive review. IEEE Trans Pattern Anal Mach Intell 46(1):150\u2013170","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"11423_CR222","doi-asserted-by":"crossref","unstructured":"Yuan L, Tay FE, Li G, Wang T, Feng J (2020) Revisiting knowledge distillation via label smoothing regularization. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 3903\u20133911","DOI":"10.1109\/CVPR42600.2020.00396"},{"issue":"1","key":"11423_CR221","doi-asserted-by":"publisher","first-page":"506","DOI":"10.1093\/bib\/bbab506","volume":"23","author":"W Yuan","year":"2022","unstructured":"Yuan W, Chen G, Chen CY-C (2022) Fusiondta: attention-based feature polymerizer and knowledge distillation for drug-target binding affinity prediction. Brief Bioinform 23(1):506","journal-title":"Brief Bioinform"},{"key":"11423_CR223","doi-asserted-by":"crossref","unstructured":"Yu R, Liu S, Ye J, Wang X (2024) Teddy: Efficient large-scale dataset distillation via Taylor-approximated matching. In: Proceedings of the European conference on computer vision (ECCV), pp 1\u201317","DOI":"10.1007\/978-3-031-72952-2_1"},{"key":"11423_CR224","doi-asserted-by":"crossref","unstructured":"Zhai X (2022) Chatgpt user experience: implications for education. Available at SSRN 4312418","DOI":"10.2139\/ssrn.4312418"},{"key":"11423_CR234","doi-asserted-by":"crossref","unstructured":"Zhang Y, Xiang T, Hospedales TM, Lu H (2018a) Deep mutual learning. In: Proceedings of the ieee conference on computer vision and pattern recognition, pp 4320\u20134328","DOI":"10.1109\/CVPR.2018.00454"},{"key":"11423_CR235","doi-asserted-by":"crossref","unstructured":"Zhang X, Xie R, Ma P (2018b) Statistical leveraging methods in big data. In: Handbook of big data analytics, pp 51\u201374","DOI":"10.1007\/978-3-319-18284-1_3"},{"key":"11423_CR229","unstructured":"Zhang T, Kishore V, Wu F, Weinberger KQ, Artzi Y (2019a) Bertscore: Evaluating text generation with bert. arXiv preprint arXiv:1904.09675"},{"key":"11423_CR231","doi-asserted-by":"publisher","unstructured":"Zhang L, Song J, Gao A, Chen J, Bao C, Ma K (2019b) Be your own teacher: improve the performance of convolutional neural networks via self distillation. In: Proceedings of the IEEE\/CVF international conference on computer vision (ICCV), pp 3712\u20133721. https:\/\/doi.org\/10.1109\/ICCV.2019.00381 . https:\/\/api.semanticscholar.org\/CorpusID:159041406","DOI":"10.1109\/ICCV.2019.00381"},{"issue":"8","key":"11423_CR225","doi-asserted-by":"publisher","first-page":"4388","DOI":"10.1109\/TPAMI.2021.3067100","volume":"44","author":"L Zhang","year":"2022","unstructured":"Zhang L, Bao C, Ma K (2022a) Self-distillation: towards efficient and compact neural networks. IEEE Trans Pattern Anal Mach Intell 44(8):4388\u20134403. https:\/\/doi.org\/10.1109\/TPAMI.2021.3067100","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"11423_CR226","doi-asserted-by":"crossref","unstructured":"Zhang H, Chen D, Wang C (2022b) Confidence-aware multi-teacher knowledge distillation. In: ICASSP 2022-2022 IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, pp 4498\u20134502","DOI":"10.1109\/ICASSP43922.2022.9747534"},{"key":"11423_CR227","doi-asserted-by":"crossref","unstructured":"Zhang H, Chen D, Wang C (2023a) Adaptive multi-teacher knowledge distillation with meta-learning. In: 2023 IEEE international conference on multimedia and expo (ICME). IEEE, pp 1943\u20131948","DOI":"10.1109\/ICME55011.2023.00333"},{"key":"11423_CR228","unstructured":"Zhang S, Dong L, Li X, Zhang S, Sun X, Wang S, Li J, Hu R, Zhang T, Wu F et al (2023b) Instruction tuning for large language models: a survey. arXiv preprint arXiv:2308.10792"},{"key":"11423_CR232","unstructured":"Zhang C, Song D, Ye Z, Gao Y (2023c) Towards the law of capacity gap in distilling language models. arXiv preprint arXiv:2311.07052"},{"key":"11423_CR233","unstructured":"Zhang X, Tian C, Yang X, Chen L, Li Z, Petzold LR (2023d) Alpacare: Instruction-tuned large language models for medical application. arXiv preprint arXiv:2310.14558"},{"key":"11423_CR230","doi-asserted-by":"crossref","unstructured":"Zhang J, Qin Y, Pi R, Zhang W, Pan R, Zhang T (2024a) Tagcos: Task-agnostic gradient clustered coreset selection for instruction tuning data. CoRR","DOI":"10.18653\/v1\/2025.findings-naacl.264"},{"key":"11423_CR237","unstructured":"Zhang X, Zhai J, Ma S, Shen C, Li T, Jiang W, Liu Y (2024b) Staff: speculative coreset selection for task-specific fine-tuning. In: The thirteenth international conference on learning representations"},{"key":"11423_CR236","unstructured":"Zhang Z, You W, Wu T, Wang X, Li J, Zhang M (2025a) A survey of generative information extraction. In: Proceedings of the 31st International conference on computational linguistics, pp 4840\u20134870"},{"key":"11423_CR238","unstructured":"Zhang K, Zhu R, Ma S et al (2025b) Kedrec-lm: A knowledge-distilled explainable drug recommendation llm. arXiv preprint arXiv:2501.01558"},{"key":"11423_CR243","unstructured":"Zhao B, Mopuri KR, Bilen H (2020) Dataset condensation with gradient matching. arXiv preprint arXiv:2006.05929"},{"key":"11423_CR240","unstructured":"Zhao B, Bilen H (2021) Dataset condensation with differentiable siamese augmentation. In: International conference on machine learning. PMLR, pp 12674\u201312685"},{"key":"11423_CR241","doi-asserted-by":"crossref","unstructured":"Zhao B, Bilen H (2023) Dataset condensation with distribution matching. In: Proceedings of the IEEE\/CVF winter conference on applications of computer vision, pp 6514\u20136523","DOI":"10.1109\/WACV56688.2023.00645"},{"key":"11423_CR239","doi-asserted-by":"publisher","DOI":"10.1016\/j.compbiomed.2023.107284","volume":"164","author":"L Zhao","year":"2023","unstructured":"Zhao L, Qian X, Guo Y, Song J, Hou J, Gong J (2023a) Mskd: Structured knowledge distillation for efficient medical image segmentation. Comput Biol Med 164:107284","journal-title":"Comput Biol Med"},{"key":"11423_CR242","doi-asserted-by":"crossref","unstructured":"Zhao G, Li G, Qin Y, Yu Y (2023b) Improved distribution matching for dataset condensation. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 7856\u20137865","DOI":"10.1109\/CVPR52729.2023.00759"},{"key":"11423_CR244","unstructured":"Zhao W, Sui X, Guo J, Hu Y, Deng Y, Zhao Y, Qin B, Che W, Chua T-S, Liu T (2025) Trade-offs in large reasoning models: an empirical analysis of deliberative and adaptive reasoning over foundational capabilities. arXiv preprint arXiv:2503.17979"},{"key":"11423_CR245","unstructured":"Zhong X, Chen B, Fang H, Gu X, Xia S-T, Yang E-H (2024a) Going beyond feature similarity: effective dataset distillation based on class-aware conditional mutual information. arXiv preprint arXiv:2412.09945"},{"key":"11423_CR246","unstructured":"Zhong X, Sun S, Gu X, Xu Z, Wang Y, Zhang M, Chen B (2024b) Efficient dataset distillation via diffusion-driven patch selection for improved generalization. arXiv preprint arXiv:2412.09959"},{"issue":"1","key":"11423_CR247","doi-asserted-by":"publisher","first-page":"295","DOI":"10.1186\/s12859-021-04200-w","volume":"22","author":"H Zhou","year":"2021","unstructured":"Zhou H, Liu Z, Lang C, Xu Y, Lin Y, Hou J (2021a) Improving the recall of biomedical named entity recognition with label re-correction and knowledge distillation. BMC Bioinform 22(1):295","journal-title":"BMC Bioinform"},{"key":"11423_CR252","unstructured":"Zhou H, Song L, Chen J, Zhou Y, Wang G, Yuan J, Zhang Q (2021b) Rethinking soft labels for knowledge distillation: a bias-variance tradeoff perspective. arXiv preprint arXiv:2102.00650"},{"key":"11423_CR255","unstructured":"Zhu Z, Hong J, Zhou J (2021c) Data-free knowledge distillation for heterogeneous federated learning. In: International conference on machine learning. PMLR, pp 12878\u201312889"},{"key":"11423_CR248","first-page":"9813","volume":"35","author":"Y Zhou","year":"2022","unstructured":"Zhou Y, Nezhadarya E, Ba J (2022) Dataset distillation using neural feature regression. Adv Neural Inf Process Syst 35:9813\u20139827","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR249","first-page":"55006","volume":"36","author":"C Zhou","year":"2023","unstructured":"Zhou C, Liu P, Xu P, Iyer S, Sun J, Mao Y, Ma X, Efrat A, Yu P, Yu L et al (2023a) Lima: Less is more for alignment. Adv Neural Inf Process Syst 36:55006\u201355021","journal-title":"Adv Neural Inf Process Syst"},{"key":"11423_CR250","unstructured":"Zhou Y, Lyu K, Rawat AS, Menon AK, Rostamizadeh A, Kumar S, Kagy J-F, Agarwal R (2023b) Distillspec: Improving speculative decoding via knowledge distillation. arXiv preprint arXiv:2310.08461"},{"key":"11423_CR251","doi-asserted-by":"crossref","unstructured":"Zhou J, Rizzo K, Tang Z, Koo PK (2024a) Uncertainty-aware genomic deep learning with knowledge distillation. bioRxiv","DOI":"10.1101\/2024.11.13.623485"},{"key":"11423_CR253","doi-asserted-by":"crossref","unstructured":"Zhou Y, Wang X, Niu Y, Shen Y, Tang L, Chen F, He B, Sun L, Wen L (2024b) Difflm: Controllable synthetic data generation via diffusion language models. arXiv preprint arXiv:2411.03250","DOI":"10.18653\/v1\/2025.findings-acl.1061"},{"key":"11423_CR254","doi-asserted-by":"crossref","unstructured":"Zhou W, Zhu X, Han Q-L, Li L, Chen X, Wen S, Xiang Y (2024c) The security of using large language models: a survey with emphasis on chatgpt. IEEE\/CAA J Automatica Sinica","DOI":"10.1109\/JAS.2024.124983"},{"key":"11423_CR256","doi-asserted-by":"crossref","unstructured":"Zhu K, Wang J, Zhou J, Wang Z, Chen H, Wang Y, Yang L, Ye W, Zhang Y, Zhenqiang\u00a0Gong N et al (2023) Promptbench: Towards evaluating the robustness of large language models on adversarial prompts. arXiv e-prints, 2306","DOI":"10.1145\/3689217.3690621"}],"container-title":["Artificial Intelligence Review"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10462-025-11423-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10462-025-11423-3","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10462-025-11423-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,27]],"date-time":"2026-01-27T04:56:43Z","timestamp":1769489803000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10462-025-11423-3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11,20]]},"references-count":254,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2026,1]]}},"alternative-id":["11423"],"URL":"https:\/\/doi.org\/10.1007\/s10462-025-11423-3","relation":{},"ISSN":["1573-7462"],"issn-type":[{"value":"1573-7462","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,11,20]]},"assertion":[{"value":"8 May 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"7 October 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"20 November 2025","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that there is no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"Not applicable.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical approval"}}],"article-number":"17"}}