{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,19]],"date-time":"2025-11-19T07:02:25Z","timestamp":1763535745139,"version":"3.37.3"},"reference-count":53,"publisher":"Springer Science and Business Media LLC","issue":"9-10","license":[{"start":{"date-parts":[[2020,8,5]],"date-time":"2020-08-05T00:00:00Z","timestamp":1596585600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,8,5]],"date-time":"2020-08-05T00:00:00Z","timestamp":1596585600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["DMR-1534431","IIS-1514053"],"award-info":[{"award-number":["DMR-1534431","IIS-1514053"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100014989","name":"the Chan Zuckerberg Initiative","doi-asserted-by":"crossref","id":[{"id":"10.13039\/100014989","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2020,9]]},"DOI":"10.1007\/s10994-020-05897-1","type":"journal-article","created":{"date-parts":[[2020,8,5]],"date-time":"2020-08-05T22:02:27Z","timestamp":1596664947000},"page":"1749-1778","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":13,"title":["Using error decay prediction to overcome practical issues of deep active learning for named entity recognition"],"prefix":"10.1007","volume":"109","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4607-936X","authenticated-orcid":false,"given":"Haw-Shiuan","family":"Chang","sequence":"first","affiliation":[]},{"given":"Shankar","family":"Vembu","sequence":"additional","affiliation":[]},{"given":"Sunil","family":"Mohan","sequence":"additional","affiliation":[]},{"given":"Rheeya","family":"Uppaal","sequence":"additional","affiliation":[]},{"given":"Andrew","family":"McCallum","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2020,8,5]]},"reference":[{"issue":"4","key":"5897_CR1","doi-asserted-by":"publisher","first-page":"605","DOI":"10.1162\/neco.1992.4.4.605","volume":"4","author":"A Si","year":"1992","unstructured":"Amari, S., Fujita, N., & Shinomoto, S. (1992). Four types of learning curves. Neural Computation, 4(4), 605\u2013618.","journal-title":"Neural Computation"},{"key":"5897_CR2","unstructured":"Bachman, P., Sordoni, A., & Trischler, A. (2017). Learning algorithms for active learning. In ICML."},{"key":"5897_CR3","unstructured":"Baldridge, J., & Osborne, M. (2004). Active learning and the total cost of annotation. In NIPS."},{"key":"5897_CR4","doi-asserted-by":"crossref","unstructured":"Bloodgood, M., & Vijay-Shanker, K. (2009). A method for stopping active learning based on stabilizing predictions and the need for user-adjustable stopping. In CoNLL.","DOI":"10.3115\/1596374.1596384"},{"key":"5897_CR5","doi-asserted-by":"crossref","unstructured":"Bodenreider, O. (2004). The unified medical language system (umls): Integrating biomedical terminology. Nucleic Acids Research, 32(suppl$$_{-}$$1), D267\u2013D270.","DOI":"10.1093\/nar\/gkh061"},{"key":"5897_CR6","doi-asserted-by":"crossref","unstructured":"Bouguelia, M. R., Bela\u00efd, Y., & Bela\u00efd, A. (2015). Stream-based active learning in the presence of label noise. In 4th International conference on pattern recognition applications and methods-ICPRAM 2015.","DOI":"10.5220\/0005178900250034"},{"issue":"8","key":"5897_CR7","doi-asserted-by":"publisher","first-page":"1307","DOI":"10.1007\/s13042-017-0645-0","volume":"9","author":"MR Bouguelia","year":"2018","unstructured":"Bouguelia, M. R., Nowaczyk, S., Santosh, K., & Verikas, A. (2018). Agreeing to disagree: Active learning with noisy labels without crowdsourcing. International Journal of Machine Learning and Cybernetics, 9(8), 1307\u20131319.","journal-title":"International Journal of Machine Learning and Cybernetics"},{"key":"5897_CR8","unstructured":"Chen, I., Johansson, F. D., & Sontag, D. (2018). Why is my classifier discriminatory? In Advances in neural information processing systems (pp. 3543\u20133554)."},{"key":"5897_CR9","unstructured":"Collobert, R., Weston, J., Bottou, L., Karlen, M., Kavukcuoglu, K., & Kuksa, P. (2011). Natural language processing (almost) from scratch. Journal of Machine Learning Research, 12(Aug), 2493\u20132537."},{"key":"5897_CR10","doi-asserted-by":"crossref","unstructured":"Culotta, A., & McCallum, A. (2005). Reducing labeling effort for structured prediction tasks. In AAAI.","DOI":"10.21236\/ADA440382"},{"issue":"19","key":"5897_CR11","doi-asserted-by":"publisher","first-page":"1767","DOI":"10.1016\/j.tcs.2010.12.054","volume":"412","author":"S Dasgupta","year":"2011","unstructured":"Dasgupta, S. (2011). Two faces of active learning. Theoretical Computer Science, 412(19), 1767\u20131781.","journal-title":"Theoretical Computer Science"},{"key":"5897_CR12","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.jbi.2013.12.006","volume":"47","author":"RI Do\u011fan","year":"2014","unstructured":"Do\u011fan, R. I., Leaman, R., & Lu, Z. (2014). NCBI disease corpus: A resource for disease name recognition and concept normalization. Journal of Biomedical Informatics, 47, 1\u201310.","journal-title":"Journal of Biomedical Informatics"},{"key":"5897_CR13","doi-asserted-by":"crossref","unstructured":"Fang, M., Li, Y., & Cohn, T. (2017). Learning how to active learn: A deep reinforcement learning approach. In EMNLP.","DOI":"10.18653\/v1\/D17-1063"},{"key":"5897_CR14","doi-asserted-by":"crossref","unstructured":"Fu, W., Wang, M., Hao, S., & Wu, X. (2018). Scalable active learning by approximated error reduction. In SIGKDD.","DOI":"10.1145\/3219819.3219954"},{"key":"5897_CR15","unstructured":"Gal, Y., Islam, R., & Ghahramani, Z. (2017). Deep bayesian active learning with image data. In ICML."},{"key":"5897_CR16","doi-asserted-by":"crossref","unstructured":"Greenberg, N., Bansal, T., Verga, P., & McCallum, A. (2018). Marginal likelihood training of bilstm-crf for biomedical named entity recognition from disjoint label sets. In EMNLP.","DOI":"10.18653\/v1\/D18-1306"},{"key":"5897_CR17","unstructured":"Guillory, A., & Bilmes, J. (2010). Interactive submodular set cover. arXiv preprint arXiv:10023345."},{"key":"5897_CR18","unstructured":"Hestness, J., Narang, S., Ardalani, N., Diamos, G., Jun, H., Kianinejad, H., et al. (2017). Deep learning scaling is predictable, empirically. arXiv preprint arXiv:171200409."},{"key":"5897_CR19","unstructured":"Khetan, A., Lipton, Z. C., & Anandkumar, A. (2018). Learning from noisy singly-labeled data. In ICLR."},{"key":"5897_CR20","unstructured":"Konyushkova, K., Sznitman, R., & Fua, P. (2017). Learning active learning from data. In NIPS."},{"key":"5897_CR21","doi-asserted-by":"crossref","unstructured":"Koshorek, O., Stanovsky, G., Zhou, Y., Srikumar, V., & Berant, J. (2019). On the limits of learning to actively learn semantic representations. In CoNLL.","DOI":"10.18653\/v1\/K19-1042"},{"key":"5897_CR22","unstructured":"Kremer, J., Sha, F., & Igel, C. (2018). Robust active label correction. In International conference on artificial intelligence and statistics."},{"key":"5897_CR23","doi-asserted-by":"crossref","unstructured":"Lakkaraju, H., Kamar, E., Caruana, R., & Horvitz, E. (2017). Identifying unknown unknowns in the open world: Representations and policies for guided exploration. In AAAI.","DOI":"10.1609\/aaai.v31i1.10821"},{"key":"5897_CR24","doi-asserted-by":"crossref","unstructured":"Lowell, D., Lipton, Z. C., & Wallace, BC. (2019). Practical obstacles to deploying active learning. In EMNLP.","DOI":"10.18653\/v1\/D19-1003"},{"issue":"5","key":"5897_CR25","doi-asserted-by":"publisher","first-page":"960","DOI":"10.1145\/185675.306789","volume":"41","author":"C Lund","year":"1994","unstructured":"Lund, C., & Yannakakis, M. (1994). On the hardness of approximating minimization problems. Journal of the ACM (JACM), 41(5), 960\u2013981.","journal-title":"Journal of the ACM (JACM)"},{"key":"5897_CR26","first-page":"281","volume":"1","author":"J MacQueen","year":"1967","unstructured":"MacQueen, J., et al. (1967). Some methods for classification and analysis of multivariate observations. Proceedings of the fifth Berkeley symposium on mathematical statistics and probability, Oakland, CA, USA, 1, 281\u2013297.","journal-title":"Proceedings of the fifth Berkeley symposium on mathematical statistics and probability, Oakland, CA, USA"},{"key":"5897_CR27","unstructured":"Mikolov, T., Sutskever, I., Chen, K., Corrado, G. S., & Dean, J. (2013). Distributed representations of words and phrases and their compositionality. In NIPS."},{"key":"5897_CR28","volume-title":"Probability and computing: Randomization and probabilistic techniques in algorithms and data analysis","author":"M Mitzenmacher","year":"2017","unstructured":"Mitzenmacher, M., & Upfal, E. (2017). Probability and computing: Randomization and probabilistic techniques in algorithms and data analysis. Oxford: Cambridge University Press."},{"key":"5897_CR29","doi-asserted-by":"crossref","unstructured":"Murty, S., Verga, P., Vilnis, L., Radovanovic, I., & McCallum, A. (2018). Hierarchical losses and new resources for fine-grained entity typing and linking. In ACL.","DOI":"10.18653\/v1\/P18-1010"},{"key":"5897_CR30","unstructured":"Mussmann, S., & Liang, P. (2018). On the relationship between data efficiency and error for uncertainty sampling. In ICML."},{"issue":"4","key":"5897_CR31","doi-asserted-by":"publisher","first-page":"770","DOI":"10.1137\/0721052","volume":"21","author":"SG Nash","year":"1984","unstructured":"Nash, S. G. (1984). Newton-type minimization via the Lanczos method. SIAM Journal on Numerical Analysis, 21(4), 770\u2013788.","journal-title":"SIAM Journal on Numerical Analysis"},{"key":"5897_CR32","unstructured":"Phillips, R., Chang, K. H., & Friedler, S. A. (2018). Interpretable active learning. In Conference on fairness, accountability and transparency."},{"key":"5897_CR33","unstructured":"Ravi, S., & Larochelle, H. (2018). Meta-learning for batch mode active learning. In ICLR workshop."},{"key":"5897_CR34","doi-asserted-by":"crossref","unstructured":"Reichart, R., Tomanek, K., Hahn, U., & Rappoport, A. (2008). Multi-task active learning for linguistic annotations. In ACL.","DOI":"10.3115\/1698381.1698399"},{"key":"5897_CR35","unstructured":"Roy, N., & McCallum, A. (2001). Toward optimal active learning through Monte Carlo estimation of error reduction. In ICML."},{"issue":"1&2","key":"5897_CR36","first-page":"125","volume":"40","author":"N Rubens","year":"2011","unstructured":"Rubens, N., Sheinman, V., Tomioka, R., & Sugiyama, M. (2011). Active learning in black-box settings. Austrian Journal of Statistics, 40(1&2), 125\u2013135.","journal-title":"Austrian Journal of Statistics"},{"key":"5897_CR37","doi-asserted-by":"crossref","unstructured":"Sculley, D. (2010). Web-scale k-means clustering. In WWW.","DOI":"10.1145\/1772690.1772862"},{"key":"5897_CR38","unstructured":"Sener, O., & Savarese, S. (2018). Active learning for convolutional neural networks: Acore-set approach. In ICLR."},{"key":"5897_CR39","unstructured":"Settles, B. (2009). Active learning literature survey. Computer Sciences Technical Report 1648, University of Wisconsin-Madison."},{"key":"5897_CR40","unstructured":"Settles, B. (2011). From theories to queries: Active learning in practice. In Active Learning and Experimental Design workshop In conjunction with AISTATS 2010."},{"key":"5897_CR41","doi-asserted-by":"crossref","unstructured":"Settles, B., & Craven, M. (2008). An analysis of active learning strategies for sequence labeling tasks. In EMNLP.","DOI":"10.3115\/1613715.1613855"},{"key":"5897_CR42","unstructured":"Settles, B., Craven, M., & Ray, S. (2008). Multiple-instance active learning. In NIPS."},{"key":"5897_CR43","doi-asserted-by":"crossref","unstructured":"Shen, Y., Yun, H., Lipton, Z. C., Kronrod, Y., & Anandkumar, A. (2018). Deep active learning for named entity recognition. In ICLR.","DOI":"10.18653\/v1\/W17-2630"},{"key":"5897_CR44","doi-asserted-by":"crossref","unstructured":"Sheng, V. S., Provost, F., & Ipeirotis, P. G. (2008). Get another label? Improving data quality and data mining using multiple, noisy labelers. In SIGKDD.","DOI":"10.1145\/1401890.1401965"},{"key":"5897_CR45","doi-asserted-by":"crossref","unstructured":"Siddhant, A., & Lipton, Z. C. (2018). Deep bayesian active learning for natural language processing: Results of a large-scale empirical study. In EMNLP.","DOI":"10.18653\/v1\/D18-1318"},{"key":"5897_CR46","doi-asserted-by":"crossref","unstructured":"Strubell, E., Verga, P., Belanger, D., & McCallum, A. (2017). Fast and accurate entity recognition with iterated dilated convolutions. In EMNLP.","DOI":"10.18653\/v1\/D17-1283"},{"key":"5897_CR47","unstructured":"Sang, E. F. T. K., & De\u00a0Meulder, F. (2003). Introduction to the CoNLL-2003 shared task: Language-independent named entity recognition. In NAACL."},{"key":"5897_CR48","doi-asserted-by":"crossref","unstructured":"Tomanek, K., & Olsson, F. (2009). A web survey on the use of active learning to support annotation of text data. In NAACL HLT 2009 workshop on active learning for natural language processing.","DOI":"10.3115\/1564131.1564140"},{"key":"5897_CR49","doi-asserted-by":"crossref","unstructured":"Wang, C., Chiticariu, L., & Li, Y. (2017a). Active learning for black-box semantic role labeling with neural factors. In IJCAI.","DOI":"10.24963\/ijcai.2017\/405"},{"key":"5897_CR50","doi-asserted-by":"crossref","unstructured":"Wang, W., Yang, N., Wei, F., Chang, B., & Zhou, M. (2017b). Gated self-matching networks for reading comprehension and question answering. In ACL.","DOI":"10.18653\/v1\/P17-1018"},{"key":"5897_CR51","unstructured":"Wei, K., Iyer, R., & Bilmes, J. (2015). Submodularity in data subset selection and active learning. In ICML."},{"issue":"5","key":"5897_CR52","doi-asserted-by":"publisher","first-page":"1095","DOI":"10.1109\/TCYB.2014.2344674","volume":"45","author":"J Zhang","year":"2015","unstructured":"Zhang, J., Wu, X., & Shengs, V. S. (2015). Active learning with imbalanced multiple noisy labeling. IEEE Transactions on Cybernetics, 45(5), 1095\u20131107.","journal-title":"IEEE Transactions on Cybernetics"},{"key":"5897_CR53","doi-asserted-by":"crossref","unstructured":"Zhao, L., Sukthankar, G., & Sukthankar, R. (2011). Incremental relabeling for active learning with noisy crowdsourced annotations. In SocialCom\/PASSAT.","DOI":"10.1109\/PASSAT\/SocialCom.2011.193"}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-020-05897-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10994-020-05897-1\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-020-05897-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,11,5]],"date-time":"2022-11-05T22:06:02Z","timestamp":1667685962000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10994-020-05897-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,8,5]]},"references-count":53,"journal-issue":{"issue":"9-10","published-print":{"date-parts":[[2020,9]]}},"alternative-id":["5897"],"URL":"https:\/\/doi.org\/10.1007\/s10994-020-05897-1","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"type":"print","value":"0885-6125"},{"type":"electronic","value":"1573-0565"}],"subject":[],"published":{"date-parts":[[2020,8,5]]},"assertion":[{"value":"15 December 2019","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"16 June 2020","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 July 2020","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"5 August 2020","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Compliance with ethical standards"}},{"value":"Any opinions, findings and conclusions or recommendations expressed in this material are those of the authors and do not necessarily reflect those of the sponsor.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}