{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T12:24:02Z","timestamp":1785587042243,"version":"3.56.0"},"reference-count":41,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2022,12,8]],"date-time":"2022-12-08T00:00:00Z","timestamp":1670457600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,12,8]],"date-time":"2022-12-08T00:00:00Z","timestamp":1670457600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100000266","name":"Engineering and Physical Sciences Research Council","doi-asserted-by":"publisher","award":["EP\/V043544\/1"],"award-info":[{"award-number":["EP\/V043544\/1"]}],"id":[{"id":"10.13039\/501100000266","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["CCF Trans. Pervasive Comp. Interact."],"published-print":{"date-parts":[[2023,3]]},"DOI":"10.1007\/s42486-022-00115-4","type":"journal-article","created":{"date-parts":[[2022,12,8]],"date-time":"2022-12-08T19:08:16Z","timestamp":1670526496000},"page":"1-12","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":11,"title":["Human-in-the-loop machine learning with applications for population health"],"prefix":"10.1007","volume":"5","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8933-2663","authenticated-orcid":false,"given":"Long","family":"Chen","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiangtao","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Bin","family":"Guo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Liming","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2022,12,8]]},"reference":[{"key":"115_CR1","doi-asserted-by":"crossref","unstructured":"Amershi, S., et al.: Effective end-user interaction with machine learning. In: Twenty-fifth AAAI conference on artificial intelligence (2011).","DOI":"10.1145\/2046396.2046416"},{"issue":"4","key":"115_CR2","first-page":"105","volume":"35","author":"S Amershi","year":"2014","unstructured":"Amershi, S., et al.: Power to the people: the role of humans in interactive machine learning. AI Mag. 35(4), 105\u2013120 (2014)","journal-title":"AI Mag."},{"key":"115_CR3","doi-asserted-by":"crossref","unstructured":"Barbosa, N.M., et al,: Rehumanized crowdsourcing: a labeling framework addressing bias and ethics in machine learning, p. 543. CHI. ACM (2019)","DOI":"10.1145\/3290605.3300773"},{"key":"115_CR4","doi-asserted-by":"crossref","unstructured":"Cheng, J., et al.: Flock: hybrid crowd-machine learning classifiers. In: Proceedings of the 18th ACM Conference on Computer Supported Cooperative Work & Social Computing, pp. 600\u2013611 (2015).","DOI":"10.1145\/2675133.2675214"},{"key":"115_CR5","doi-asserted-by":"crossref","unstructured":"Chu, X., et al.: Katara: a data cleaning system powered by knowledge bases and crowdsourcing. In: Proceedings of the 2015 ACM SIGMOD International Conference on Management of Data, pp. 1247\u20131261 (2015).","DOI":"10.1145\/2723372.2749431"},{"key":"115_CR6","doi-asserted-by":"publisher","first-page":"20","DOI":"10.1109\/MCG.2019.2922592","volume":"39","author":"S Das","year":"2019","unstructured":"Das, S., et al.: BEAMES: interactive multi-model steering, selection, and inspection for regression tasks. IEEE Comput Graph Appl 39, 20\u201332 (2019)","journal-title":"IEEE Comput Graph Appl"},{"issue":"4","key":"115_CR7","doi-asserted-by":"publisher","first-page":"666","DOI":"10.1109\/TPAMI.2015.2439285","volume":"38","author":"J Deng","year":"2016","unstructured":"Deng, J., et al.: Leveraging the wisdom of the crowd for fine-grained recognition. IEEE Trans. Pattern Anal. Mach. Intell. 38(4), 666\u2013676 (2016)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"issue":"2","key":"115_CR8","doi-asserted-by":"publisher","first-page":"8","DOI":"10.1145\/3185517","volume":"8","author":"JJ Dudley","year":"2018","unstructured":"Dudley, J.J., et al.: A review of user interface design for interactive machine learning. ACM Trans. Interact. Intell. Syst. 8(2), 8 (2018)","journal-title":"ACM Trans. Interact. Intell. Syst."},{"issue":"47","key":"115_CR9","doi-asserted-by":"publisher","first-page":"249","DOI":"10.1016\/j.arcontrol.2019.03.004","volume":"1","author":"C Emmanouilidis","year":"2019","unstructured":"Emmanouilidis, C., Pistofidis, P., Bertoncelj, L., Katsouros, V., Fournaris, A., Koulamas, C., Ruiz-Carcel, C.: Enabling the human in the loop: linked data and knowledge in industrial cyber-physical systems. Annu. Rev. Control. 1(47), 249\u2013265 (2019)","journal-title":"Annu. Rev. Control."},{"key":"115_CR10","doi-asserted-by":"crossref","unstructured":"Feng, Y., Wang, J., Wang, Y., Helal, S.: Completing missing prevalence rates for multiple chronic diseases by jointly leveraging both intra-and inter-disease population health data correlations. In: Proceedings of the web conference, pp. 183\u2013193. (2021).","DOI":"10.1145\/3442381.3449811"},{"issue":"10","key":"115_CR11","doi-asserted-by":"publisher","first-page":"2020","DOI":"10.1109\/TMC.2014.2385097","volume":"14","author":"B Guo","year":"2014","unstructured":"Guo, B., et al.: FlierMeet: a mobile crowdsensing system for cross-space public information reposting, tagging, and sharing. IEEE Trans. Mob. Comput. 14(10), 2020\u20132033 (2014)","journal-title":"IEEE Trans. Mob. Comput."},{"key":"115_CR12","doi-asserted-by":"crossref","unstructured":"Guo, B., et al.: From crowdsourcing to crowdmining: using implicit human intelligence for better understanding of crowdsourced data. World Wide Web, pp. 1\u201325 (2019).","DOI":"10.1007\/s11280-019-00718-5"},{"key":"115_CR13","doi-asserted-by":"crossref","unstructured":"Honeycutt, D., Nourani, M., Ragan, E.: Soliciting human-in-the-loop user feedback for interactive machine learning reduces user trust and impressions of model accuracy. In: Proceedings of the AAAI Conference on Human Computation and Crowdsourcing, vol. 8, pp. 63\u201372 (2020).","DOI":"10.1609\/hcomp.v8i1.7464"},{"key":"115_CR14","unstructured":"Houlsby, N., Husz\u00e1r, F., Ghahramani, Z., Lengyel, M.: Bayesian active learning for classification and preference learning. arXiv preprint arXiv:1112.5745 (2011)."},{"issue":"6","key":"115_CR15","first-page":"1","volume":"14","author":"J Howe","year":"2006","unstructured":"Howe, J.: The rise of crowdsourcing. Wired Mag. 14(6), 1\u20134 (2006)","journal-title":"Wired Mag."},{"issue":"29","key":"115_CR16","doi-asserted-by":"publisher","first-page":"5320","DOI":"10.1002\/sim.2968","volume":"26","author":"W Jiang","year":"2007","unstructured":"Jiang, W., Simon, R.: A comparison of bootstrap methods and an adjusted bootstrap approach for estimating the prediction error in microarray classification. Stat. Med. 26(29), 5320\u20135334 (2007)","journal-title":"Stat. Med."},{"key":"115_CR17","unstructured":"Kamar, E.: Directions in hybrid intelligence: complementing AI systems with human intelligence. In: IJCAI, pp. 4070\u20134073 (2016)."},{"key":"115_CR18","doi-asserted-by":"crossref","unstructured":"Li, G., et al.: Crowdsourced data management: Overview and challenges. In: Proceedings of the 2017 ACM International Conference on Management of Data, pp. 1711\u20131716. ACM, (2017).","DOI":"10.1145\/3035918.3054776"},{"key":"115_CR19","unstructured":"Liu, Y., et al.: Doubly active learning: when active learning meets active crowdsourcing. AAAI (2018)."},{"key":"115_CR20","doi-asserted-by":"crossref","unstructured":"Lu, J., Yang, L., Mac Namee, B., Zhang, Y.: A rationale-centric framework for human-in-the-loop machine learning. arXiv preprint arXiv:2203.12918 (2022).","DOI":"10.18653\/v1\/2022.acl-long.481"},{"key":"115_CR21","volume-title":"Human-in-the-Loop Machine Learning: Active Learning and Annotation for Human-Centered AI","author":"RM Monarch","year":"2021","unstructured":"Monarch, R.M.: Human-in-the-Loop Machine Learning: Active Learning and Annotation for Human-Centered AI. Simon and Schuster, New York (2021)"},{"issue":"4","key":"115_CR22","doi-asserted-by":"publisher","first-page":"909","DOI":"10.1007\/s11136-014-0820-7","volume":"24","author":"RE Mujica-Mota","year":"2015","unstructured":"Mujica-Mota, R.E., Roberts, M., Abel, G., Elliott, M., Lyratzopoulos, G., Roland, M., Campbell, J.: Common patterns of morbidity and multi-morbidity and their impact on health-related quality of life: evidence from a national survey. Qual. Life Res. 24(4), 909\u2013918 (2015)","journal-title":"Qual. Life Res."},{"key":"115_CR23","doi-asserted-by":"crossref","unstructured":"Nushi, B., et al.: Crowd access path optimization: diversity matters. In: Third AAAI Conference on Human Computation and Crowdsourcing (2015)","DOI":"10.1609\/hcomp.v3i1.13228"},{"key":"115_CR24","doi-asserted-by":"publisher","DOI":"10.23915\/distill.00010","author":"C Olah","year":"2018","unstructured":"Olah, C., et al.: The building blocks of interpretability. Distill (2018). https:\/\/doi.org\/10.23915\/distill.00010","journal-title":"Distill"},{"key":"115_CR25","doi-asserted-by":"crossref","unstructured":"Ribeiro, M.T., et al.: Why should I trust you? Explaining the predictions of any classifier. In: Proceedings of the 22nd ACM SIGKDD international conference on knowledge discovery and data mining, pp. 1135\u20131144. ACM (2016).","DOI":"10.1145\/2939672.2939778"},{"issue":"5","key":"115_CR26","first-page":"93","volume":"51","author":"G Riccardo","year":"2019","unstructured":"Riccardo, G., et al.: A survey of methods for explaining black box models. ACM Comput. Surv. (CSUR) 51(5), 93 (2019)","journal-title":"ACM Comput. Surv. (CSUR)"},{"key":"115_CR27","doi-asserted-by":"crossref","unstructured":"Ruan, W., et al.: Global robustness evaluation of deep neural networks with provable guarantees for the hamming distance. IJCAI (2019).","DOI":"10.24963\/ijcai.2019\/824"},{"key":"115_CR28","unstructured":"Teso, S., Kersting, K.: Why should i trust interactive learners? Explaining interactive queries of classifiers to users. arXiv preprint, 5 (2018)."},{"key":"115_CR29","doi-asserted-by":"crossref","unstructured":"Tong, Y., et al.: Crowdcleaner: data cleaning for multi-version data on the web via crowdsourcing. In: Data engineering (ICDE), 2014 IEEE 30th international conference on, pp. 1182\u20131185. IEEE (2014).","DOI":"10.1109\/ICDE.2014.6816736"},{"key":"115_CR30","doi-asserted-by":"publisher","first-page":"517","DOI":"10.1613\/jair.5175","volume":"56","author":"M Venanzi","year":"2016","unstructured":"Venanzi, M., et al.: Time-sensitive Bayesian information aggregation for crowdsourcing systems. J. Artif. Intell. Res. 56, 517\u2013545 (2016)","journal-title":"J. Artif. Intell. Res."},{"key":"115_CR31","doi-asserted-by":"crossref","unstructured":"Wang J, et al.. Crowd-Assisted Machine Learning: Current Issues and Future Directions. Computer. 2019 13;52(1):46\u201353.","DOI":"10.1109\/MC.2018.2890174"},{"key":"115_CR32","doi-asserted-by":"crossref","unstructured":"Wu X, Xiao L, Sun Y, Zhang J, Ma T, He L. A survey of human-in-the-loop for machine learning. Future Gener. Comput. Syst. 18 (2022).","DOI":"10.1016\/j.future.2022.05.014"},{"key":"115_CR33","doi-asserted-by":"crossref","unstructured":"Xin, D., Ma, L., Liu, J., Macke, S., Song, S., Parameswaran, A.: Helix: accelerating human-in-the-loop machine learning. arXiv preprint arXiv:1808.01095. (2018).","DOI":"10.1145\/3209889.3209897"},{"key":"115_CR34","unstructured":"Xu, C., et al.: Data cleaning: overview and emerging challenges. In: Proceedings of the 2016 international conference on Management of Data, pp. 2201\u20132206. ACM (2016)."},{"key":"115_CR35","unstructured":"Yosinski, J., et al.: Understanding neural networks through deep visualization. In: ICML Workshop on Deep Learning (2015)."},{"issue":"1","key":"115_CR36","doi-asserted-by":"publisher","first-page":"27","DOI":"10.1631\/FITEE.1700808","volume":"19","author":"QS Zhang","year":"2018","unstructured":"Zhang, Q.S., Zhu, S.C.: Visual interpretability for deep learning: a survey. Front. Inf. Technol. Electron. Eng. 19(1), 27\u201339 (2018)","journal-title":"Front. Inf. Technol. Electron. Eng."},{"key":"115_CR37","doi-asserted-by":"publisher","first-page":"135","DOI":"10.1109\/TKDE.2018.2881185","volume":"32","author":"C Zhang","year":"2018","unstructured":"Zhang, C., et al.: Reducing uncertainty of schema matching via crowdsourcing with accuracy rates. IEEE Trans. Knowl. Data Eng. 32, 135\u2013151 (2018)","journal-title":"IEEE Trans. Knowl. Data Eng."},{"key":"115_CR38","doi-asserted-by":"crossref","unstructured":"Zhang, R., et al.: Leveraging human guidance for deep reinforcement learning tasks. In: Proceedings of the 28th International Joint Conference on Artificial Intelligence (IJCAI) (2019).","DOI":"10.24963\/ijcai.2019\/884"},{"issue":"1","key":"115_CR39","doi-asserted-by":"publisher","first-page":"44","DOI":"10.1093\/nsr\/nwx106","volume":"5","author":"ZH Zhou","year":"2017","unstructured":"Zhou, Z.H.: A brief introduction to weakly supervised learning. Natl. Sci. Rev. 5(1), 44\u201353 (2017)","journal-title":"Natl. Sci. Rev."},{"key":"115_CR40","doi-asserted-by":"crossref","unstructured":"Zhuang, Y, et al.: Hike: a hybrid human-machine method for entity alignment in large-scale knowledge bases. In: Proceedings of the 2017 ACM on Conference on Information and Knowledge Management, pp 1917\u20131926. ACM (2017).","DOI":"10.1145\/3132847.3132912"},{"key":"115_CR41","doi-asserted-by":"crossref","unstructured":"Zou, J.Y., et al.: Crowdsourcing feature discovery via adaptively chosen comparisons. In: Proceedings of the 31st International Conference on Machine Learning, Lille, France (2015)","DOI":"10.1609\/hcomp.v3i1.13231"}],"container-title":["CCF Transactions on Pervasive Computing and Interaction"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s42486-022-00115-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s42486-022-00115-4\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s42486-022-00115-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,2,24]],"date-time":"2023-02-24T09:06:12Z","timestamp":1677229572000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s42486-022-00115-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,12,8]]},"references-count":41,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2023,3]]}},"alternative-id":["115"],"URL":"https:\/\/doi.org\/10.1007\/s42486-022-00115-4","relation":{},"ISSN":["2524-521X","2524-5228"],"issn-type":[{"value":"2524-521X","type":"print"},{"value":"2524-5228","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,12,8]]},"assertion":[{"value":"29 April 2022","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"16 August 2022","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"8 December 2022","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}