{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T11:39:18Z","timestamp":1774438758512,"version":"3.50.1"},"reference-count":94,"publisher":"Springer Science and Business Media LLC","issue":"9","license":[{"start":{"date-parts":[[2025,6,4]],"date-time":"2025-06-04T00:00:00Z","timestamp":1748995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"},{"start":{"date-parts":[[2025,6,4]],"date-time":"2025-06-04T00:00:00Z","timestamp":1748995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"}],"funder":[{"DOI":"10.13039\/501100001809","name":"Natural Science Foundation of China","doi-asserted-by":"crossref","award":["62302487"],"award-info":[{"award-number":["62302487"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100001809","name":"Natural Science Foundation of China","doi-asserted-by":"crossref","award":["62302487"],"award-info":[{"award-number":["62302487"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100001809","name":"Natural Science Foundation of China","doi-asserted-by":"crossref","award":["62302487"],"award-info":[{"award-number":["62302487"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100001809","name":"Natural Science Foundation of China","doi-asserted-by":"crossref","award":["62302487"],"award-info":[{"award-number":["62302487"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100001809","name":"Natural Science Foundation of China","doi-asserted-by":"crossref","award":["62302487"],"award-info":[{"award-number":["62302487"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100001809","name":"Natural Science Foundation of China","doi-asserted-by":"crossref","award":["62302487"],"award-info":[{"award-number":["62302487"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100001809","name":"Natural Science Foundation of China","doi-asserted-by":"crossref","award":["62302487"],"award-info":[{"award-number":["62302487"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100001809","name":"Natural Science Foundation of China","doi-asserted-by":"crossref","award":["62302487"],"award-info":[{"award-number":["62302487"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]},{"name":"Improvement Project of Chinese Academy of Sciences","award":["GSZXKYZB2025007"],"award-info":[{"award-number":["GSZXKYZB2025007"]}]},{"name":"Improvement Project of Chinese Academy of Sciences","award":["GSZXKYZB2025007"],"award-info":[{"award-number":["GSZXKYZB2025007"]}]},{"name":"Improvement Project of Chinese Academy of Sciences","award":["GSZXKYZB2025007"],"award-info":[{"award-number":["GSZXKYZB2025007"]}]},{"name":"Improvement Project of Chinese Academy of Sciences","award":["GSZXKYZB2025007"],"award-info":[{"award-number":["GSZXKYZB2025007"]}]},{"name":"Improvement Project of Chinese Academy of Sciences","award":["GSZXKYZB2025007"],"award-info":[{"award-number":["GSZXKYZB2025007"]}]},{"name":"Improvement Project of Chinese Academy of Sciences","award":["GSZXKYZB2025007"],"award-info":[{"award-number":["GSZXKYZB2025007"]}]},{"name":"Improvement Project of Chinese Academy of Sciences","award":["GSZXKYZB2025007"],"award-info":[{"award-number":["GSZXKYZB2025007"]}]},{"name":"Improvement Project of Chinese Academy of Sciences","award":["GSZXKYZB2025007"],"award-info":[{"award-number":["GSZXKYZB2025007"]}]},{"name":"Science and Technology Innovation Program of Hunan Province","award":["2022RC4006"],"award-info":[{"award-number":["2022RC4006"]}]},{"name":"Science and Technology Innovation Program of Hunan Province","award":["2022RC4006"],"award-info":[{"award-number":["2022RC4006"]}]},{"name":"Science and Technology Innovation Program of Hunan Province","award":["2022RC4006"],"award-info":[{"award-number":["2022RC4006"]}]},{"name":"Science and Technology Innovation Program of Hunan Province","award":["2022RC4006"],"award-info":[{"award-number":["2022RC4006"]}]},{"name":"Science and Technology Innovation Program of Hunan Province","award":["2022RC4006"],"award-info":[{"award-number":["2022RC4006"]}]},{"name":"Science and Technology Innovation Program of Hunan Province","award":["2022RC4006"],"award-info":[{"award-number":["2022RC4006"]}]},{"name":"Science and Technology Innovation Program of Hunan Province","award":["2022RC4006"],"award-info":[{"award-number":["2022RC4006"]}]},{"name":"Science and Technology Innovation Program of Hunan Province","award":["2022RC4006"],"award-info":[{"award-number":["2022RC4006"]}]},{"name":"Innovation Funding of ICT, CAS"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Artif Intell Rev"],"DOI":"10.1007\/s10462-025-11255-1","type":"journal-article","created":{"date-parts":[[2025,6,4]],"date-time":"2025-06-04T01:24:26Z","timestamp":1749000266000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["Agent-in-the-loop to distill expert knowledge into artificial intelligence models: a survey"],"prefix":"10.1007","volume":"58","author":[{"given":"Jiayuan","family":"Gao","sequence":"first","affiliation":[]},{"given":"Yingwei","family":"Zhang","sequence":"additional","affiliation":[]},{"given":"Yiqiang","family":"Chen","sequence":"additional","affiliation":[]},{"given":"Yihan","family":"Dong","sequence":"additional","affiliation":[]},{"given":"Yuanzhe","family":"Chen","sequence":"additional","affiliation":[]},{"given":"Shuchao","family":"Song","sequence":"additional","affiliation":[]},{"given":"Boshi","family":"Tang","sequence":"additional","affiliation":[]},{"given":"Yang","family":"Gu","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,6,4]]},"reference":[{"key":"11255_CR1","doi-asserted-by":"crossref","unstructured":"Ahn Y, Lin Y-R, Xu P, Dai Z (2023) Escape: countering systematic errors from machine\u2019s blind spots via interactive visual analysis. In: Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems, pp 1\u201316","DOI":"10.1145\/3544548.3581373"},{"issue":"4","key":"11255_CR2","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3569504","volume":"6","author":"R Arakawa","year":"2023","unstructured":"Arakawa R, Yakura H, Mollyn V, Nie S, Russell E, DeMeo DP, Reddy HA, Maytin AK, Carroll BT, Lehman JF et al (2023) Prism-tracker: A framework for multimodal procedure tracking using wearable sensors and state transition information with user-driven handling of errors and uncertainty. Proceedings of the ACM on Interactive, Mobile, Wearable and Ubiquitous Technologies 6(4):1\u201327","journal-title":"Proceedings of the ACM on Interactive, Mobile, Wearable and Ubiquitous Technologies"},{"key":"11255_CR3","doi-asserted-by":"crossref","unstructured":"Arous I, Dolamic L, Yang J, Bhardwaj A, Cuccu G, Cudr\u00e9-Mauroux P (2021) Marta: leveraging human rationales for explainable text classification. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol 35, pp. 5868\u20135876","DOI":"10.1609\/aaai.v35i7.16734"},{"key":"11255_CR4","volume-title":"Unequal justice: lawyers and social change in modern America","author":"JS Auerbach","year":"1977","unstructured":"Auerbach JS (1977) Unequal justice: lawyers and social change in modern America. Oxford University Press, Oxford"},{"key":"11255_CR5","doi-asserted-by":"publisher","first-page":"662","DOI":"10.1162\/tacl_a_00338","volume":"8","author":"M Bartolo","year":"2020","unstructured":"Bartolo M, Roberts A, Welbl J, Riedel S, Stenetorp P (2020) Beat the ai: investigating adversarial human annotation for reading comprehension. Trans Assoc Comput Linguist 8:662\u2013678","journal-title":"Trans Assoc Comput Linguist"},{"key":"11255_CR6","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/7503.003.0022","author":"S Ben-David","year":"2006","unstructured":"Ben-David S, Blitzer J, Crammer K, Pereira F (2006) Analysis of representations for domain adaptation. Adv Neural Inf Process Syst. https:\/\/doi.org\/10.7551\/mitpress\/7503.003.0022","journal-title":"Adv Neural Inf Process Syst"},{"key":"11255_CR7","unstructured":"Bommasani R, Hudson DA, Adeli E, Altman R, Arora S, Arx S, Bernstein MS, Bohg J, Bosselut A, Brunskill E, et al (2021) On the opportunities and risks of foundation models. Preprint at https:\/\/arxiv.org\/abs\/quant-ph\/2108.07258"},{"key":"11255_CR8","first-page":"1877","volume":"33","author":"T Brown","year":"2020","unstructured":"Brown T, Mann B, Ryder N, Subbiah M, Kaplan JD, Dhariwal P, Neelakantan A, Shyam P, Sastry G, Askell A et al (2020) Language models are few-shot learners. Adv Neural Inf Process Syst 33:1877\u20131901","journal-title":"Adv Neural Inf Process Syst"},{"issue":"1","key":"11255_CR9","first-page":"43","volume":"43","author":"RP Buckley","year":"2021","unstructured":"Buckley RP, Zetzsche DA, Arner DW, Tang BW (2021) Regulating artificial intelligence in finance: putting the human in the loop. Sydney Law Rev 43(1):43\u201381","journal-title":"Sydney Law Rev"},{"key":"11255_CR10","doi-asserted-by":"publisher","first-page":"102062","DOI":"10.1016\/j.media.2021.102062","volume":"71","author":"S Budd","year":"2021","unstructured":"Budd S, Robinson EC, Kainz B (2021) A survey on active learning and human-in-the-loop deep learning for medical image analysis. Med Image Anal 71:102062","journal-title":"Med Image Anal"},{"key":"11255_CR11","doi-asserted-by":"crossref","unstructured":"Cai CJ, Reif E, Hegde N, Hipp J, Kim B, Smilkov D, Wattenberg M, Viegas F, Corrado GS, Stumpe MC, et al (2019) Human-centered tools for coping with imperfect algorithms during medical decision-making. In: Proceedings of the 2019 Chi Conference on Human Factors in Computing Systems, pp 1\u201314","DOI":"10.1145\/3290605.3300234"},{"issue":"4","key":"11255_CR12","first-page":"1","volume":"54","author":"K Chen","year":"2021","unstructured":"Chen K, Zhang D, Yao L, Guo B, Yu Z, Liu Y (2021a) Deep learning for sensor-based human activity recognition: overview, challenges, and opportunities. ACM Comput Surv (CSUR) 54(4):1\u201340","journal-title":"ACM Comput Surv (CSUR)"},{"key":"11255_CR13","doi-asserted-by":"crossref","unstructured":"Chen X, Jiang M, Zhao Q (2021b) Leveraging human attention in novel object captioning. In: International Joint Conference on Artificial Intelligence","DOI":"10.24963\/ijcai.2021\/86"},{"issue":"1","key":"11255_CR14","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3580812","volume":"7","author":"S Cho","year":"2023","unstructured":"Cho S, Kim Y, Jang J, Hwang I (2023) Ai-to-human actuation: Boosting unmodified ai\u2019s robustness by proactively inducing favorable human sensing conditions. Proceedings of the ACM on Interactive, Mobile, Wearable and Ubiquitous Technologies 7(1):1\u201332","journal-title":"Proceedings of the ACM on Interactive, Mobile, Wearable and Ubiquitous Technologies"},{"key":"11255_CR15","doi-asserted-by":"crossref","unstructured":"Crochepierre L, Boudjeloud-Assala L, Barbesant V (2022) Interactive reinforcement learning for symbolic regression from multi-format human-preference feedbacks. In: 31st International Joint Conference on Artificial Intelligence (IJCAI 2022)","DOI":"10.24963\/ijcai.2022\/849"},{"key":"11255_CR16","doi-asserted-by":"crossref","unstructured":"Cui Y, Koppol P, Admoni H, Niekum S, Simmons R, Steinfeld A, Fitzgerald T (2021) Understanding the relationship between interactions and outcomes in human-in-the-loop machine learning. In: International Joint Conference on Artificial Intelligence","DOI":"10.24963\/ijcai.2021\/599"},{"key":"11255_CR17","unstructured":"Cui J, Ning M, Li Z, Chen B, Yan Y, Li H, Ling B, Tian Y, Yuan L (2024) Chatlaw: a multi-agent collaborative legal assistant with knowledge graph enhanced mixture-of-experts large language model.\u00a0Preprint at https:\/\/arxiv.org\/abs\/quant-ph\/2306.16092"},{"key":"11255_CR18","unstructured":"Dai H, Liu Z, Liao W, Huang X, Cao Y, Wu Z, Zhao L, Xu S, Liu W, Liu N et al (2023) Auggpt: leveraging chatgpt for text data augmentation.\u00a0Preprint at https:\/\/arxiv.org\/abs\/quant-ph\/2302.13007"},{"key":"11255_CR63","doi-asserted-by":"publisher","first-page":"119898","DOI":"10.1016\/j.ins.2023.119898","volume":"655","author":"J Del Ser","year":"2024","unstructured":"Del Ser J, Barredo-Arrieta A, D\u00edaz-Rodr\u00edguez N, Herrera F, Saranti A, Holzinger A (2024) On generating trustworthy counterfactual explanations. Inf Sci 655:119898. https:\/\/doi.org\/10.1016\/j.ins.2023.119898","journal-title":"Inf Sci"},{"key":"11255_CR19","doi-asserted-by":"crossref","unstructured":"Ding B, Qin C, Liu L, Bing L, Joty SR, Li BA (2022) Is gpt-3 a good data annotator? In: Annual meeting of the Association for Computational Linguistics. https:\/\/api.semanticscholar.org\/CorpusID:254877171","DOI":"10.18653\/v1\/2023.acl-long.626"},{"key":"11255_CR20","first-page":"1","volume-title":"International colloquium on automata, languages, and programming","author":"C Dwork","year":"2006","unstructured":"Dwork C (2006) Differential privacy. International colloquium on automata, languages, and programming. Springer, Cham, pp 1\u201312"},{"key":"11255_CR21","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2305.16381","author":"Y Fan","year":"2024","unstructured":"Fan Y, Watkins O, Du Y, Liu H, Ryu M, Boutilier C, Abbeel P, Ghavamzadeh M, Lee K, Lee K (2024) Reinforcement learning for fine-tuning text-to-image diffusion models. Adv Neural Inf Process Syst. https:\/\/doi.org\/10.48550\/arXiv.2305.16381","journal-title":"Adv Neural Inf Process Syst"},{"key":"11255_CR22","unstructured":"Gao J, Pi R, Lin Y, Xu H, Ye J, Wu Z, Zhang W, Liang X, Li Z, Kong L (2023) Self-guided noise-free data generation for efficient zero-shot learning. In: The Twelfth International Conference on Learning Representations"},{"key":"11255_CR23","doi-asserted-by":"crossref","unstructured":"Gao J, Zhang Y, Chen Y, Zhang T, Tang B, Wang X (2024) Unsupervised human activity recognition via large language models and iterative evolution. In: ICASSP 2024-2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), IEEE, pp 91\u201395","DOI":"10.1109\/ICASSP48485.2024.10446819"},{"issue":"6","key":"11255_CR24","doi-asserted-by":"publisher","first-page":"1789","DOI":"10.1007\/s11263-021-01453-z","volume":"129","author":"J Gou","year":"2021","unstructured":"Gou J, Yu B, Maybank SJ, Tao D (2021) Knowledge distillation: a survey. Int J Comput Vis 129(6):1789\u20131819","journal-title":"Int J Comput Vis"},{"key":"11255_CR25","unstructured":"Guo S, Zhang B, Liu T, Liu T, Khalman M, Llinares F, Rame A, Mesnard T, Zhao Y, Piot B, et al (2024) Direct language model alignment from online ai feedback.\u00a0Preprint at https:\/\/arxiv.org\/abs\/quant-ph\/2402.04792"},{"key":"11255_CR26","doi-asserted-by":"crossref","unstructured":"Hancock B, Bordes A, Mazare P-E, Weston J (2019) Learning from dialogue after deployment: feed yourself, chatbot! Preprint at\u00a0https:\/\/arxiv.org\/abs\/quant-ph\/1901.05415","DOI":"10.18653\/v1\/P19-1358"},{"key":"11255_CR27","doi-asserted-by":"publisher","unstructured":"He Z, Ribeiro MT, Khani F (2023a) Targeted data generation: finding and fixing model weaknesses. In: Rogers A, Boyd-Graber J, Okazaki N (eds) Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Vol 1: Long Papers), Association for Computational Linguistics, Toronto,\u00a0pp 8506\u20138520.\u00a0https:\/\/doi.org\/10.18653\/v1\/2023.acl-long.474. https:\/\/aclanthology.org\/2023.acl-long.474","DOI":"10.18653\/v1\/2023.acl-long.474"},{"key":"11255_CR28","doi-asserted-by":"crossref","unstructured":"He X, Lin Z, Gong Y, Jin A, Zhang H, Lin C, Jiao J, Yiu SM, Duan N, Chen W, et al (2023b) Annollm: making large language models to be better crowdsourced annotators.\u00a0Preprint at https:\/\/arxiv.org\/abs\/quant-ph\/2303.16854","DOI":"10.18653\/v1\/2024.naacl-industry.15"},{"key":"11255_CR29","doi-asserted-by":"crossref","unstructured":"Hemmer P, Schellhammer S, V\u00f6ssing M, Jakubik J, Satzger G (2022) Forming effective human-ai teams: building machine learning models that complement the capabilities of multiple experts.\u00a0Preprint at https:\/\/arxiv.org\/abs\/quant-ph\/2206.07948\u00a0","DOI":"10.24963\/ijcai.2022\/344"},{"key":"11255_CR30","unstructured":"Hinton G (2015) Distilling the knowledge in a neural network. Preprint at\u00a0https:\/\/arxiv.org\/abs\/quant-ph\/1503.02531"},{"issue":"3","key":"11255_CR31","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3550294","volume":"6","author":"SK Hiremath","year":"2022","unstructured":"Hiremath SK, Nishimura Y, Chernova S, Pl\u00f6tz T (2022) Bootstrapping human activity recognition systems for smart homes from scratch. Proceedings of the ACM on Interactive, Mobile, Wearable and Ubiquitous Technologies 6(3):1\u201327","journal-title":"Proceedings of the ACM on Interactive, Mobile, Wearable and Ubiquitous Technologies"},{"issue":"2","key":"11255_CR32","doi-asserted-by":"publisher","first-page":"119","DOI":"10.1007\/s40708-016-0042-6","volume":"3","author":"A Holzinger","year":"2016","unstructured":"Holzinger A (2016) Interactive machine learning for health informatics: when do we need the human-in-the-loop? Brain Inform 3(2):119\u2013131","journal-title":"Brain Inform"},{"key":"11255_CR33","doi-asserted-by":"publisher","unstructured":"Hsieh C-Y, Li C-L, Yeh C-K, Nakhost H, Fujii Y, Ratner A, Krishna R, Lee C-Y, Pfister T (2023a) Distilling step-by-step! outperforming larger language models with less training data and smaller model sizes. In: Rogers A, Boyd-Graber J, Okazaki N (eds) Findings of the Association for Computational Linguistics: ACL 2023, Association for Computational Linguistics, Toronto,\u00a0pp 8003\u20138017.\u00a0https:\/\/doi.org\/10.18653\/v1\/2023.findings-acl.507. https:\/\/aclanthology.org\/2023.findings-acl.507","DOI":"10.18653\/v1\/2023.findings-acl.507"},{"key":"11255_CR34","doi-asserted-by":"crossref","unstructured":"Hsieh C (2023b) Human-centred multimodal deep learning models for chest x-ray diagnosis. In: Proceedings of the Thirty-Second International Joint Conference on Artificial Intelligence, pp 7085\u20137086","DOI":"10.24963\/ijcai.2023\/817"},{"key":"11255_CR35","doi-asserted-by":"crossref","unstructured":"Kath H, Gouv\u00eaa TS, Sonntag D (2023) A human-in-the-loop tool for annotating passive acoustic monitoring datasets. In: Proceedings of the 32nd International Joint Conference on Artificial Intelligence, IJCAI","DOI":"10.24963\/ijcai.2023\/835"},{"key":"11255_CR36","unstructured":"Kenton JDMWC, Toutanova LK (2019) Bert: pre-training of deep bidirectional transformers for language understanding. In: Proceedings of NAACL-HLT, pp 4171\u20134186"},{"key":"11255_CR37","doi-asserted-by":"crossref","unstructured":"Klie J-C, Castilho RE, Gurevych I (2020) From zero to hero: Human-in-the-loop entity linking in low resource domains. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp 6982\u20136993","DOI":"10.18653\/v1\/2020.acl-main.624"},{"key":"11255_CR38","unstructured":"Klissarov M, D\u2019Oro P, Sodhani S, Raileanu R, Bacon P-L, Vincent P, Zhang A, Henaff M (2023) Motif: intrinsic motivation from artificial intelligence feedback"},{"key":"11255_CR39","doi-asserted-by":"crossref","unstructured":"Koppol P, Admoni H, Simmons RG (2021) Interaction considerations in learning from humans. In: IJCAI, pp 283\u2013291","DOI":"10.24963\/ijcai.2021\/40"},{"key":"11255_CR40","unstructured":"Krizhevsky A, Sutskever I, Hinton GE (2012) Imagenet classification with deep convolutional neural networks. Adv Neural Inf Process Syst 25"},{"key":"11255_CR41","unstructured":"Kwon M, Michael S (2023) Reward design with language models. In: International Conference on Learning Representations (ICLR)"},{"issue":"7553","key":"11255_CR42","doi-asserted-by":"publisher","first-page":"436","DOI":"10.1038\/nature14539","volume":"521","author":"Y LeCun","year":"2015","unstructured":"LeCun Y, Bengio Y, Hinton G (2015) Deep learning. Nature 521(7553):436\u2013444","journal-title":"Nature"},{"issue":"12","key":"11255_CR43","doi-asserted-by":"publisher","first-page":"2006","DOI":"10.14778\/3137765.3137833","volume":"10","author":"G Li","year":"2017","unstructured":"Li G (2017) Human-in-the-loop data integration. Proc VLDB Endow 10(12):2006\u20132017","journal-title":"Proc VLDB Endow"},{"key":"11255_CR44","unstructured":"Li H, Dong Q, Tang Z, Wang C, Zhang X, Huang H, Huang S, Huang X, Huang Z, Zhang D et al (2024) Synthetic data (almost) from scratch: generalized instruction tuning for language models. Preprint at https:\/\/arxiv.org\/abs\/quant-ph\/2402.13064"},{"key":"11255_CR46","doi-asserted-by":"crossref","unstructured":"Liu Z, Guo Y, Mahmud J (2021) When and why a model fails? a human-in-the-loop error detection framework for sentiment analysis. In: Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: human language technologies: Industry Papers, pp 170\u2013177","DOI":"10.18653\/v1\/2021.naacl-industry.22"},{"issue":"7","key":"11255_CR45","doi-asserted-by":"publisher","first-page":"1237","DOI":"10.1093\/jamia\/ocad072","volume":"30","author":"S Liu","year":"2023","unstructured":"Liu S, Wright AP, Patterson BL, Wanderer JP, Turer RW, Nelson SD, McCoy AB, Sittig DF, Wright A (2023) Using ai-generated suggestions from chatgpt to optimize clinical decision support. J Am Med Inform Assoc 30(7):1237\u20131245","journal-title":"J Am Med Inform Assoc"},{"key":"11255_CR47","doi-asserted-by":"publisher","first-page":"4441","DOI":"10.1109\/LRA.2023.3284380","volume":"8","author":"Y Long","year":"2023","unstructured":"Long Y, Wei W, Huang T, Wang Y, Dou Q (2023) Human-in-the-loop embodied intelligence with interactive simulation environment for surgical robot learning. IEEE Robot Autom Lett 8:4441\u20138","journal-title":"IEEE Robot Autom Lett"},{"key":"11255_CR48","doi-asserted-by":"crossref","unstructured":"Lu F, Wang W, Luo Y, Zhu Z, Sun Q, Xu B, Shi H, Gao S, Li Q, Song Y, et al (2024) Miko: multimodal intention knowledge distillation from large language models for social-media commonsense discovery. In: Proceedings of the 32nd ACM International Conference on Multimedia, pp 3303\u20133312","DOI":"10.1145\/3664647.3681339"},{"key":"11255_CR49","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbi.2024.104600","volume":"150","author":"JM Metsch","year":"2024","unstructured":"Metsch JM, Saranti A, Angerschmid A, Pfeifer B, Klemt V, Holzinger A, Hauschild A-C (2024) Clarus: An interactive explainable ai platform for manual counterfactuals in graph neural networks. J Biomed Inform 150:104600. https:\/\/doi.org\/10.1016\/j.jbi.2024.104600","journal-title":"J Biomed Inform"},{"key":"11255_CR50","unstructured":"Mondorf P, Plank B (2024) Beyond accuracy: evaluating the reasoning behavior of large language models\u2014a survey. Preprint at https:\/\/arxiv.org\/abs\/quant-ph\/2404.01869"},{"issue":"4","key":"11255_CR51","doi-asserted-by":"publisher","first-page":"3005","DOI":"10.1007\/s10462-022-10246-w","volume":"56","author":"E Mosqueira-Rey","year":"2023","unstructured":"Mosqueira-Rey E, Hern\u00e1ndez-Pereira E, Alonso-R\u00edos D, Bobes-Bascar\u00e1n J, Fern\u00e1ndez-Leal \u00c1 (2023a) Human-in-the-loop machine learning: a state of the art. Artif Intell Rev 56(4):3005\u20133054","journal-title":"Artif Intell Rev"},{"key":"11255_CR52","doi-asserted-by":"publisher","first-page":"2597","DOI":"10.1007\/s00521-023-09197-2","volume":"36","author":"E Mosqueira-Rey","year":"2023","unstructured":"Mosqueira-Rey E, Hern\u00e1ndez-Pereira E, Bobes-Bascar\u00e1n J, Alonso-R\u00edos D, P\u00e9rez-S\u00e1nchez A, Fern\u00e1ndez-Leal \u00c1, Moret-Bonillo V, Vidal-\u00cdnsua Y, V\u00e1zquez-Rivera F (2023b) Addressing the data bottleneck in medical deep learning models using a human-in-the-loop machine learning approach. Neural Comput Appl 36:2597","journal-title":"Neural Comput Appl"},{"key":"11255_CR53","doi-asserted-by":"crossref","unstructured":"Oh SW, Lee J-Y, Xu N, Kim SJ (2019) Fast user-guided video object segmentation by interaction-and-propagation networks. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp 5247\u20135256","DOI":"10.1109\/CVPR.2019.00539"},{"key":"11255_CR54","first-page":"27730","volume":"35","author":"L Ouyang","year":"2022","unstructured":"Ouyang L, Wu J, Jiang X, Almeida D, Wainwright C, Mishkin P, Zhang C, Agarwal S, Slama K, Ray A et al (2022) Training language models to follow instructions with human feedback. Adv Neural Inf Process Syst 35:27730\u201327744","journal-title":"Adv Neural Inf Process Syst"},{"issue":"1","key":"11255_CR55","first-page":"67","volume":"31","author":"W Pan","year":"2020","unstructured":"Pan W, Wang X, Song M, Chen C (2020) Survey on generating adversarial examples. Ruan Jian Xue Bao\/J Softw 31(1):67\u201381 (in Chinese)","journal-title":"Ruan Jian Xue Bao\/J Softw"},{"key":"11255_CR56","first-page":"11338","volume":"36","author":"JS Park","year":"2023","unstructured":"Park JS, Hessel J, Chandu K, Liang PP, Lu X, West P, Yu Y, Huang Q, Gao J, Farhadi A et al (2023) Localized symbolic knowledge distillation for visual commonsense models. Adv Neural Inf Process Syst 36:11338\u201311352","journal-title":"Adv Neural Inf Process Syst"},{"key":"11255_CR57","doi-asserted-by":"crossref","unstructured":"Qian K, Raman PC, Li Y, Popa L (2020) Partner: Human-in-the-loop entity name understanding with deep learning. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol 34, pp 13634\u201313635","DOI":"10.1609\/aaai.v34i09.7104"},{"key":"11255_CR58","unstructured":"Radford A, Narasimhan K, Salimans T, Sutskever I, et al (2018) Improving language understanding by generative pre-training."},{"key":"11255_CR59","doi-asserted-by":"publisher","DOI":"10.1613\/jair.1.15348","author":"CO Retzlaff","year":"2024","unstructured":"Retzlaff CO, Das S, Wayllace C, Mousavi P, Afshari M, Yang T, Saranti A, Angerschmid A, Taylor ME, Holzinger A (2024) Human-in-the-loop reinforcement learning: a survey and position on requirements, challenges, and opportunities. J Artif Int Res. https:\/\/doi.org\/10.1613\/jair.1.15348","journal-title":"J Artif Int Res"},{"key":"11255_CR60","doi-asserted-by":"publisher","DOI":"10.1101\/644146","author":"J Roels","year":"2019","unstructured":"Roels J, Vernaillen F, Kremer A, Gon\u00e7alves A, Aelterman J, Luong HQ, Goossens B, Philips W, Lippens S, Saeys Y (2019) A \u2018human-in-the-loop\u2019approach for semi-automated image restoration in electron microscopy. BioRxiv. https:\/\/doi.org\/10.1101\/644146","journal-title":"BioRxiv"},{"key":"11255_CR61","doi-asserted-by":"publisher","unstructured":"Sahu G, Vechtomova O, Bahdanau D, Laradji I (2023) Promptmix: a class boundary augmentation method for large language model distillation. In: Bouamor H, Pino J, Bali K. (eds) Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, Association for Computational Linguistics, Singapore,\u00a0pp 5316\u20135327 https:\/\/doi.org\/10.18653\/v1\/2023.emnlp-main.323. https:\/\/aclanthology.org\/2023.emnlp-main.323","DOI":"10.18653\/v1\/2023.emnlp-main.323"},{"issue":"1","key":"11255_CR62","doi-asserted-by":"publisher","first-page":"6668","DOI":"10.1038\/s41598-023-33500-9","volume":"13","author":"A-D Samaras","year":"2023","unstructured":"Samaras A-D, Moustakidis S, Apostolopoulos ID, Papandrianos N, Papageorgiou E (2023) Classification models for assessing coronary artery disease instances using clinical and biometric data: an explainable man-in-the-loop approach. Scientific Rep 13(1):6668","journal-title":"Scientific Rep"},{"key":"11255_CR64","unstructured":"Settles B (2009) Active learning literature survey"},{"key":"11255_CR65","first-page":"3008","volume":"33","author":"N Stiennon","year":"2020","unstructured":"Stiennon N, Ouyang L, Wu J, Ziegler D, Lowe R, Voss C, Radford A, Amodei D, Christiano PF (2020) Learning to summarize with human feedback. Adv Neural Inf Process Syst 33:3008\u20133021","journal-title":"Adv Neural Inf Process Syst"},{"key":"11255_CR66","first-page":"2511","volume":"36","author":"Z Sun","year":"2024","unstructured":"Sun Z, Shen Y, Zhou Q, Zhang H, Chen Z, Cox D, Yang Y, Gan C (2024) Principle-driven self-alignment of language models from scratch with minimal human supervision. Adv Neural Inf Process Syst 36:2511","journal-title":"Adv Neural Inf Process Syst"},{"key":"11255_CR67","doi-asserted-by":"crossref","unstructured":"Tchemeube RB, Ens J, Plut C, Pasquier P, Safi M, Grabit Y, Rolland J-B (2023) Evaluating human-ai interaction via usability, user experience and acceptance measures for mmm-c: a creative ai system for music composition. In: Proceedings of the Thirty-Second International Joint Conference on Artificial Intelligence, pp 5769\u20135778","DOI":"10.24963\/ijcai.2023\/640"},{"key":"11255_CR68","unstructured":"Touvron H, Lavril T, Izacard G, Martinet X, Lachaux M-A, Lacroix T, Rozi\u00e8re B, Goyal N, Hambro E, Azhar F, et al (2023) Llama: open and efficient foundation language models. Preprint at\u00a0https:\/\/arxiv.org\/abs\/quant-ph\/302.13971"},{"key":"11255_CR69","volume-title":"The rights revolution in the twentieth century","author":"MV Tushnet","year":"2009","unstructured":"Tushnet MV (2009) The rights revolution in the twentieth century. American Historical Association, Washington"},{"key":"11255_CR70","first-page":"1","volume":"30","author":"A Vaswani","year":"2017","unstructured":"Vaswani A (2017) Attention is all you need. Adv Neural Inf Process Syst 30:1","journal-title":"Adv Neural Inf Process Syst"},{"key":"11255_CR71","doi-asserted-by":"publisher","first-page":"387","DOI":"10.1162\/tacl_a_00279","volume":"7","author":"E Wallace","year":"2019","unstructured":"Wallace E, Rodriguez P, Feng S, Yamada I, Boyd-Graber J (2019) Trick me if you can: human-in-the-loop generation of adversarial examples for question answering. Trans Assoc Comput Linguist 7:387\u2013401","journal-title":"Trans Assoc Comput Linguist"},{"key":"11255_CR74","doi-asserted-by":"publisher","unstructured":"Wang S, Liu Y, Xu Y, Zhu C, Zeng M (2021) Want to reduce labeling cost? GPT-3 can help. In: Moens M.-F, Huang X, Specia L, Yih SW-T (eds) Findings of the Association for Computational Linguistics: EMNLP 2021, Association for Computational Linguistics, Punta Cana, pp 4195\u20134205. https:\/\/doi.org\/10.18653\/v1\/2021.findings-emnlp.354. https:\/\/aclanthology.org\/2021.findings-emnlp.354","DOI":"10.18653\/v1\/2021.findings-emnlp.354"},{"key":"11255_CR72","first-page":"8052","volume":"35","author":"J Wang","year":"2022","unstructured":"Wang J, Lan C, Liu C, Ouyang Y, Qin T, Lu W, Chen Y, Zeng W, Yu P (2022) Generalizing to unseen domains: a survey on domain generalization. IEEE Trans Knowl Data Eng 35:8052","journal-title":"IEEE Trans Knowl Data Eng"},{"issue":"1","key":"11255_CR73","first-page":"1","volume":"7","author":"Y Wang","year":"2023","unstructured":"Wang Y, Yu Z, Liu S, Zhou Z, Guo B (2023a) Genie in the model: Automatic generation of human-in-the-loop deep neural networks for mobile applications. Proceedings of the ACM on Interactive, Mobile, Wearable and Ubiquitous Technologies 7(1):1\u201329","journal-title":"Proceedings of the ACM on Interactive, Mobile, Wearable and Ubiquitous Technologies"},{"key":"11255_CR75","doi-asserted-by":"publisher","unstructured":"Wang Y, Kordi Y, Mishra S, Liu A, Smith NA, Khashabi D, Hajishirzi H (2023b) Self-instruct: aligning language models with self-generated instructions. In: Rogers A, Boyd-Graber J, Okazaki N (eds) Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Vol 1: Long Papers), Association for Computational Linguistics, Toronto,\u00a0pp 13484\u201313508.\u00a0https:\/\/doi.org\/10.18653\/v1\/2023.acl-long.754. https:\/\/aclanthology.org\/2023.acl-long.754","DOI":"10.18653\/v1\/2023.acl-long.754"},{"key":"11255_CR76","doi-asserted-by":"crossref","unstructured":"Wang X, Kim H, Rahman S, Mitra K, Miao Z (2024) Human-llm collaborative annotation through effective verification of llm labels. In: Proceedings of the CHI Conference on Human Factors in Computing Systems, pp 1\u201321","DOI":"10.1145\/3613904.3641960"},{"key":"11255_CR77","doi-asserted-by":"crossref","unstructured":"Weber T, Hu\u00dfmann H, Han Z, Matthes S, Liu Y (2020) Draw with me: human-in-the-loop for image restoration. In: Proceedings of the 25th International Conference on Intelligent User Interfaces, pp 243\u2013253","DOI":"10.1145\/3377325.3377509"},{"key":"11255_CR78","doi-asserted-by":"crossref","unstructured":"Wei J, Xie H, Chang C, Yang X (2022) Fine-tuning Deep Neural Networks by Interactively Refining the 2D Latent Space of Ambiguous Images. In: International Joint Conference on Artificial Intelligence","DOI":"10.24963\/ijcai.2022\/861"},{"key":"11255_CR80","doi-asserted-by":"crossref","unstructured":"Wu J, Harrison C, Bigham JP, Laput G (2020) Automated class discovery and one-shot interactions for acoustic activity recognition. In: Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems, pp 1\u201314","DOI":"10.1145\/3313831.3376875"},{"key":"11255_CR79","doi-asserted-by":"publisher","first-page":"364","DOI":"10.1016\/j.future.2022.05.014","volume":"135","author":"X Wu","year":"2022","unstructured":"Wu X, Xiao L, Sun Y, Zhang J, Ma T, He L (2022) A survey of human-in-the-loop for machine learning. Future Gener Comput Syst 135:364\u2013381. https:\/\/doi.org\/10.1016\/j.future.2022.05.014","journal-title":"Future Gener Comput Syst"},{"key":"11255_CR81","doi-asserted-by":"crossref","unstructured":"Xin D, Ma L, Liu J, Macke S, Song S, Parameswaran A (2018) Accelerating human-in-the-loop machine learning: challenges and opportunities. In: Proceedings of the second workshop on data management for end-to-end machine learning, pp 1\u20134","DOI":"10.1145\/3209889.3209897"},{"key":"11255_CR83","doi-asserted-by":"crossref","unstructured":"Xu X, Gong J, Brum C, Liang L, Suh B, Gupta SK, Agarwal Y, Lindsey L, Kang R, Shahsavari B, et al (2022) Enabling hand gesture customization on wrist-worn devices. In: Proceedings of the 2022 CHI Conference on Human Factors in Computing Systems, pp 1\u201319","DOI":"10.1145\/3491102.3501904"},{"key":"11255_CR84","doi-asserted-by":"publisher","unstructured":"Xu C, Guo D, Duan N, McAuley J (2023) Baize: an open-source chat model with parameter-efficient tuning on self-chat data. In: Bouamor H, Pino J, Bali K (eds) Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, Association for Computational Linguistics, Singapore,\u00a0pp. 6268\u20136278.\u00a0https:\/\/doi.org\/10.18653\/v1\/2023.emnlp-main.385. https:\/\/aclanthology.org\/2023.emnlp-main.385","DOI":"10.18653\/v1\/2023.emnlp-main.385"},{"key":"11255_CR82","unstructured":"Xu X, Li M, Tao C, Shen T, Cheng R, Li J, Xu C, Tao D, Zhou T (2024) A survey on knowledge distillation of large language models. Preprint at https:\/\/arxiv.org\/abs\/quant-ph\/2402.13116"},{"key":"11255_CR85","doi-asserted-by":"crossref","unstructured":"Yao Z, Li X, Gao J, Sadler B, Sun H (2019a) Interactive semantic parsing for if-then recipes via hierarchical reinforcement learning. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol 33, pp 2547\u20132554","DOI":"10.1609\/aaai.v33i01.33012547"},{"key":"11255_CR86","doi-asserted-by":"crossref","unstructured":"Yao Z, Su Y, Sun H, Yih W-T (2019b) Model-based interactive semantic parsing: a unified formulation and a text-to-sql case study. In: 2019 Conference on Empirical Methods in Natural Language Processing (EMNLP\u201919)","DOI":"10.18653\/v1\/D19-1547"},{"key":"11255_CR87","doi-asserted-by":"publisher","unstructured":"Ye J, Gao J, Li Q, Xu H, Feng J, Wu Z, Yu T, Kong L (2022) ZeroGen: efficient zero-shot learning via dataset generation. In: Goldberg Y, Kozareva Z, Zhang Y (eds) Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing, Association for Computational Linguistics, Abu Dhabi, pp 11653\u201311669.\u00a0https:\/\/doi.org\/10.18653\/v1\/2022.emnlp-main.801. https:\/\/aclanthology.org\/2022.emnlp-main.801","DOI":"10.18653\/v1\/2022.emnlp-main.801"},{"key":"11255_CR88","unstructured":"Ye W, Zhang Y, Wang M, Wang S, Gu X, Abbeel P, Gao Y (2023) Foundation reinforcement learning: towards embodied generalist agents with foundation prior assistance.\u00a0Preprint at https:\/\/arxiv.org\/abs\/quant-ph\/2310.02635"},{"key":"11255_CR89","unstructured":"Yu F, Seff A, Zhang Y, Song S, Funkhouser T, Xiao J (2015) Lsun: construction of a large-scale image dataset using deep learning with humans in the loop.\u00a0Preprint at https:\/\/arxiv.org\/abs\/quant-ph\/1506.03365"},{"key":"11255_CR91","doi-asserted-by":"crossref","unstructured":"Zhang S, He L, Dragut E, Vucetic S (2019) How to invest my time: lessons from human-in-the-loop entity extraction. In: Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining, pp 2305\u20132313","DOI":"10.1145\/3292500.3330773"},{"issue":"3","key":"11255_CR90","doi-asserted-by":"publisher","first-page":"107","DOI":"10.1145\/3446776","volume":"64","author":"C Zhang","year":"2021","unstructured":"Zhang C, Bengio S, Hardt M, Recht B, Vinyals O (2021) Understanding deep learning (still) requires rethinking generalization. Commun ACM 64(3):107\u2013115","journal-title":"Commun ACM"},{"key":"11255_CR92","unstructured":"Zheng C, Zhou H, Meng F, Zhou J, Huang M (2023) Large language models are not robust multiple choice selectors. In: The Twelfth International Conference on Learning Representations"},{"issue":"1","key":"11255_CR93","doi-asserted-by":"publisher","first-page":"43","DOI":"10.1109\/JPROC.2020.3004555","volume":"109","author":"F Zhuang","year":"2020","unstructured":"Zhuang F, Qi Z, Duan K, Xi D, Zhu Y, Zhu H, Xiong H, He Q (2020) A comprehensive survey on transfer learning. Proc IEEE 109(1):43\u201376","journal-title":"Proc IEEE"},{"key":"11255_CR94","unstructured":"Ziegler DM, Stiennon N, Wu J, Brown TB, Radford A, Amodei D, Christiano P, Irving G (2019) Fine-tuning language models from human preferences.\u00a0Preprint at https:\/\/arxiv.org\/abs\/quant-ph\/1909.08593"}],"container-title":["Artificial Intelligence Review"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10462-025-11255-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10462-025-11255-1\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10462-025-11255-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,6]],"date-time":"2025-09-06T17:30:27Z","timestamp":1757179827000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10462-025-11255-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,4]]},"references-count":94,"journal-issue":{"issue":"9","published-online":{"date-parts":[[2025,9]]}},"alternative-id":["11255"],"URL":"https:\/\/doi.org\/10.1007\/s10462-025-11255-1","relation":{"has-preprint":[{"id-type":"doi","id":"10.21203\/rs.3.rs-5130511\/v1","asserted-by":"object"}]},"ISSN":["1573-7462"],"issn-type":[{"value":"1573-7462","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,6,4]]},"assertion":[{"value":"2 May 2025","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"4 June 2025","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no Conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"266"}}