{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,11]],"date-time":"2026-01-11T01:50:09Z","timestamp":1768096209631,"version":"3.49.0"},"reference-count":48,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2022,9,24]],"date-time":"2022-09-24T00:00:00Z","timestamp":1663977600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,9,24]],"date-time":"2022-09-24T00:00:00Z","timestamp":1663977600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Nature Science Foundation of China","doi-asserted-by":"crossref","award":["62076178"],"award-info":[{"award-number":["62076178"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100006606","name":"Natural Science Foundation of Tianjin City","doi-asserted-by":"publisher","award":["19ZXAZNGX00050"],"award-info":[{"award-number":["19ZXAZNGX00050"]}],"id":[{"id":"10.13039\/501100006606","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Zhijiang Fund","award":["2019KB0AB03"],"award-info":[{"award-number":["2019KB0AB03"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Neural Comput & Applic"],"published-print":{"date-parts":[[2023,1]]},"DOI":"10.1007\/s00521-022-07733-0","type":"journal-article","created":{"date-parts":[[2022,9,24]],"date-time":"2022-09-24T18:03:37Z","timestamp":1664042617000},"page":"993-1007","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":8,"title":["Investigating annotation noise for named entity recognition"],"prefix":"10.1007","volume":"35","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0693-435X","authenticated-orcid":false,"given":"Yu","family":"Zhu","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yingchun","family":"Ye","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mengyang","family":"Li","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ji","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ou","family":"Wu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2022,9,24]]},"reference":[{"key":"7733_CR1","doi-asserted-by":"crossref","unstructured":"Aguilar G, L\u00f3pez-Monroy AP, Gonz\u00e1lez FA, et al (2019) Modeling noisiness to recognize named entities using multitask neural networks on social media. Preprint at arXiv:1906.04129","DOI":"10.18653\/v1\/N18-1127"},{"key":"7733_CR2","unstructured":"Akbik A, Blythe D, Vollgraf R (2018) Contextual string embeddings for sequence labeling. In: Proceedings of the 27th international conference on computational linguistics, pp 1638\u20131649"},{"key":"7733_CR3","unstructured":"Akbik A, Bergmann T, Blythe D, et al (2019) FLAIR: An easy-to-use framework for state-of-the-art NLP. In: Proceedings of the 2019 annual conference of the North American chapter of the association for computational Linguistics, pp 54\u201359"},{"key":"7733_CR4","unstructured":"Apratim B. BSMario . (2018) Long-term on-board prediction of people in traffic scenes under uncertainty. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 4194\u20134202"},{"key":"7733_CR5","doi-asserted-by":"crossref","unstructured":"Derczynski L, Nichols E, van Erp M, et al (2017) Results of the WNUT2017 shared task on novel and emerging entity recognition. In: Proceedings of the 3rd workshop on noisy user-generated text, pp 140\u2013147","DOI":"10.18653\/v1\/W17-4418"},{"key":"7733_CR6","unstructured":"Devlin J, Chang MW, Lee K, et al (2019) Bert: pre-training of deep bidirectional transformers for language understanding. In: Proceedings of the 2019 annual conference of the North American chapter of the association for computational linguistics, pp 4171\u20134186"},{"issue":"7","key":"7733_CR7","doi-asserted-by":"publisher","first-page":"1716","DOI":"10.1109\/TNNLS.2016.2546956","volume":"28","author":"Y Duan","year":"2017","unstructured":"Duan Y, Wu O (2017) Learning with auxiliary less-noisy labels. IEEE Tans Neur Netw Learn Syst 28(7):1716\u20131721. https:\/\/doi.org\/10.1109\/TNNLS.2016.2546956","journal-title":"IEEE Tans Neur Netw Learn Syst"},{"key":"7733_CR8","unstructured":"Gal Y, Ghahramani Z (2016) Dropout as a bayesian approximation: Representing model uncertainty in deep learning. In: Proceedings of the 33th international conference on machine learning, pp 1050\u20131059"},{"issue":"5\u20136","key":"7733_CR9","doi-asserted-by":"publisher","first-page":"602","DOI":"10.1016\/j.neunet.2005.06.042","volume":"18","author":"A Graves","year":"2005","unstructured":"Graves A, Schmidhuber J (2005) Framewise phoneme classification with bidirectional lstm and other neural network architectures. Neu Netw 18(5\u20136):602\u2013610. https:\/\/doi.org\/10.1016\/j.neunet.2005.06.042","journal-title":"Neu Netw"},{"key":"7733_CR10","doi-asserted-by":"crossref","unstructured":"Gui T, Ma R, Zhang Q, et al (2019) Cnn-based chinese ner with lexicon rethinking. In: In Proceedings of the 28th international joint conference on artificial intelligence, pp 4982\u20134988","DOI":"10.24963\/ijcai.2019\/692"},{"key":"7733_CR11","doi-asserted-by":"crossref","unstructured":"Guo Q, Guo Y (2022) Lexicon enhanced Chinese named entity recognition with pointer network. neural computing and applications","DOI":"10.1007\/s00521-022-07287-1"},{"key":"7733_CR12","doi-asserted-by":"crossref","unstructured":"Guo Q, Qiu X, Liu P, et al (2019) Star-transformer. In: Proceedings of the 2019 conference of the North American chapter of the association for computational linguistics: human language technologies, volume 1 (long and short papers), pp 1315\u20131325","DOI":"10.18653\/v1\/N19-1133"},{"key":"7733_CR13","unstructured":"Hang Y, Bocao D, Xipeng Q (2019) Tener: adapting transformer encoder for name entity recognition. Preprint at arXiv:1911.04474"},{"issue":"2","key":"7733_CR14","doi-asserted-by":"publisher","first-page":"371","DOI":"10.1007\/s00521-012-0922-5","volume":"23","author":"Z Hao","year":"2013","unstructured":"Hao Z, Wang H, Cai R et al (2013) Product named entity recognition for Chinese query questions based on a skip-chain crf model. Neur Computi Appl 23(2):371\u2013379. https:\/\/doi.org\/10.1007\/s00521-012-0922-5","journal-title":"Neur Computi Appl"},{"key":"7733_CR15","doi-asserted-by":"crossref","unstructured":"Huang J, Qu L, Jia R, et al (2019) O2u-net: a simple noisy label detection approach for deep neural networks. In: Proceedings of the IEEE international conference on computer vision, pp 3326\u20133334","DOI":"10.1109\/ICCV.2019.00342"},{"key":"7733_CR16","doi-asserted-by":"crossref","unstructured":"Jenni S, Favaro P (2018) Deep bilevel learning. In: Proceedings of the 15th European conference on computer vision, pp 618\u2013633","DOI":"10.1007\/978-3-030-01249-6_38"},{"key":"7733_CR17","doi-asserted-by":"crossref","unstructured":"Jindal I, Pressel D, Lester B, et al (2019) An effective label noise model for DNN text classification. In: Proceedings of the 2019 annual conference of the North American chapter of the association for computational linguistics","DOI":"10.18653\/v1\/N19-1328"},{"key":"7733_CR18","unstructured":"Kendall A, Gal Y (2017) What uncertainties do we need in Bayesian deep learning for computer vision? In: advances in neural information processing systems, pp 5574\u20135584"},{"key":"7733_CR19","unstructured":"Krizhevsky A, Sutskever I, Hinton G (2012) Imagenet classification with deep convolutional neural networks. In: Proceedings of the 26th annual conference on neural information processing Systems, pp 1097\u20131105"},{"key":"7733_CR20","unstructured":"Kun L, Yao F, Chuanqi T, et al (2021) Noisy-labeled ner with confidence estimation. In: Proceedings of the 2021 annual conference of the North American chapter of the association for computational linguistics"},{"key":"7733_CR21","unstructured":"Lee J, Bahri Y, Novak R, et al (2018) Deep neural networks as gaussian processes. In: Proceedings of the 6st international conference on learning representations"},{"issue":"4","key":"7733_CR22","doi-asserted-by":"publisher","first-page":"1790","DOI":"10.1109\/TKDE.2020.2981329","volume":"33","author":"J Li","year":"2020","unstructured":"Li J, Sun A, Ma Y (2020) Neural named entity boundary detection. IEEE Trans Knowl Data Eng 33(4):1790\u20131795","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"7733_CR23","doi-asserted-by":"crossref","unstructured":"Liang C, Yu Y, Jiang H, et al (2020) Bond: Bert-assisted open-domain named entity recognition with distant supervision. In: Proceedings of the 26th ACM SIGKDD international conference on knowledge discovery & data mining, pp 1054\u20131064","DOI":"10.1145\/3394486.3403149"},{"issue":"4","key":"7733_CR24","doi-asserted-by":"publisher","first-page":"1171","DOI":"10.1007\/s00521-020-05038-8","volume":"33","author":"Jw Liu","year":"2021","unstructured":"Liu Jw, Ren Zp, Lu Rk et al (2021) Gmm discriminant analysis with noisy label for each class. Neur Comput Appl 33(4):1171\u20131191","journal-title":"Neur Comput Appl"},{"key":"7733_CR25","doi-asserted-by":"crossref","unstructured":"Ma X, Hovy E (2016) End-to-end sequence labeling via bi-directional lstm-cnns-crf. In: Proceedings of the 54th annual meeting of the association for computational linguistics","DOI":"10.18653\/v1\/P16-1101"},{"key":"7733_CR26","unstructured":"Mikolov T, Chen K, Corrado G, et al (2013) Efficient estimation of word representations in vector space. In: Proceedings of the 1st international conference on learning representations, Scottsdale, Arizona, USA, May 2-4, 2013, workshop track proceedings"},{"issue":"6","key":"7733_CR27","doi-asserted-by":"publisher","first-page":"4905","DOI":"10.1007\/s00521-021-06680-6","volume":"34","author":"Y Nie","year":"2022","unstructured":"Nie Y, Zhang Y, Peng Y et al (2022) Borrowing wisdom from world: modeling rich external knowledge for Chinese named entity recognition. Neur Comput Appl 34(6):4905\u20134922","journal-title":"Neur Comput Appl"},{"key":"7733_CR28","unstructured":"Northcutt CG, Athalye A, Mueller J (2021) Pervasive label errors in test sets destabilize machine learning benchmarks. Preprint at arXiv:2103.14749"},{"key":"7733_CR29","unstructured":"Panchendrarajan R, Amaresan A (2018) Bidirectional LSTM-CRF for named entity recognition. In: Proceedings of the 32nd Pacific Asia conference on language, information and computation, Hong Kong"},{"key":"7733_CR30","doi-asserted-by":"crossref","unstructured":"Pennington J, Socher R, Manning C (2014) GloVe: Global vectors for word representation. In: Proceedings of the 2014 conference on empirical methods in natural language processing, pp 1532\u20131543","DOI":"10.3115\/v1\/D14-1162"},{"key":"7733_CR31","doi-asserted-by":"crossref","unstructured":"Peters ME, Ammar W, Bhagavatula C, et al (2017) Semi-supervised sequence tagging with bidirectional language models. In: Proceedings of the 55th annual meeting of the association for computational linguistics, pp 1756\u20131765","DOI":"10.18653\/v1\/P17-1161"},{"issue":"2","key":"7733_CR32","doi-asserted-by":"publisher","first-page":"165","DOI":"10.1007\/s10994-013-5411-2","volume":"95","author":"F Rodrigues","year":"2014","unstructured":"Rodrigues F, Pereira F, Ribeiro B (2014) Sequence labeling with multiple annotators. Mach Learn 95(2):165\u2013181. https:\/\/doi.org\/10.1007\/s10994-013-5411-2","journal-title":"Mach Learn"},{"key":"7733_CR33","unstructured":"Sanh V, Debut L, Chaumond J, et al (2019) Distilbert, a distilled version of bert: smaller, faster, cheaper and lighter. arXiv preprint arXiv:1910.01108"},{"key":"7733_CR34","doi-asserted-by":"crossref","unstructured":"Shang J, Liu L, Ren X, et al (2018) Learning named entity tagger using domain-specific dictionary. Preprint at arXiv:1809.03599","DOI":"10.18653\/v1\/D18-1230"},{"key":"7733_CR35","doi-asserted-by":"crossref","unstructured":"Shang Y, Huang HY, Mao X, et al (2020) Are noisy sentences useless for distant supervised relation extraction? In: Proceedings of the AAAI conference on artificial intelligence, pp 8799\u20138806","DOI":"10.1609\/aaai.v34i05.6407"},{"key":"7733_CR36","unstructured":"Shu J, Xie Q, Yi L, et al (2019) Meta-weight-net: Learning an explicit mapping for sample weighting. In: Proceedings of the 33th annual conference on neural information lrocessing systems, pp 1917\u20131928"},{"key":"7733_CR37","first-page":"142","volume":"2003","author":"EF Tjong Kim Sang","year":"2003","unstructured":"Tjong Kim Sang EF, De Meulder F (2003) Introduction to the CoNLL-2003 shared task: language-independent named entity recognition. Proc Seventh Conf Nat Language Learn at HLT-NAACL 2003:142\u2013147","journal-title":"Proc Seventh Conf Nat Language Learn at HLT-NAACL"},{"key":"7733_CR38","unstructured":"Vaswani A, Shazeer N, Parmar N, et al (2017) Attention is all you need. In: Advances in neural information processing systems, pp 5998\u20136008"},{"issue":"105","key":"7733_CR39","doi-asserted-by":"publisher","first-page":"842","DOI":"10.1016\/j.knosys.2020.105842","volume":"197","author":"J Wang","year":"2020","unstructured":"Wang J, Xu W, Fu X et al (2020) Astral: adversarial trained lstm-cnn for named entity recognition. Knowl Based Syst 197(105):842. https:\/\/doi.org\/10.1016\/j.knosys.2020.105842","journal-title":"Knowl Based Syst"},{"key":"7733_CR40","doi-asserted-by":"crossref","unstructured":"Wang Y, Liu W, Ma X, et al (2018) Iterative learning with open-set noisy labels. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 8688\u20138696","DOI":"10.1109\/CVPR.2018.00906"},{"key":"7733_CR41","doi-asserted-by":"crossref","unstructured":"Wang Z, Shang J, Liu L, et al (2019) Crossweigh: training named entity tagger from imperfect annotations. In: Proceedings of the 2019 conference on empirical methods in natural language processing and the 9th international joint conference on natural language processing, pp 5157\u20135166","DOI":"10.18653\/v1\/D19-1519"},{"issue":"107","key":"7733_CR42","doi-asserted-by":"publisher","first-page":"636","DOI":"10.1016\/j.patcog.2020.107636","volume":"110","author":"W Wei","year":"2021","unstructured":"Wei W, Wang Z, Mao X et al (2021) Position-aware self-attention based neural sequence labeling. Pattern Recogn 110(107):636. https:\/\/doi.org\/10.1016\/j.patcog.2020.107636","journal-title":"Pattern Recogn"},{"key":"7733_CR43","doi-asserted-by":"crossref","unstructured":"Xiao Y, Wang WY (2019) Quantifying uncertainties in natural language processing tasks. In: Proceedings of the AAAI conference on artificial intelligence, pp 7322\u20137329","DOI":"10.1609\/aaai.v33i01.33017322"},{"key":"7733_CR44","unstructured":"Xu Z, Qian X, Zhang Y, et al (2008) Crf-based hybrid model for word segmentation, ner and even pos tagging. In: Proceedings of the sixth SIGHAN workshop on Chinese language processing, pp 167\u2013170"},{"key":"7733_CR45","doi-asserted-by":"crossref","unstructured":"Zhai F, Potdar S, Xiang B, et al (2017) Neural models for sequence chunking. In: Proceedings of the AAAI conference on artificial intelligence","DOI":"10.1609\/aaai.v31i1.10995"},{"key":"7733_CR46","doi-asserted-by":"crossref","unstructured":"Zhang X, Wu X, Chen F, et al (2020) Self-paced robust learning for leveraging clean labels in noisy data. In: Proceedings of the AAAI conference on artificial intelligence, pp 6853\u20136860","DOI":"10.1609\/aaai.v34i04.6166"},{"key":"7733_CR47","doi-asserted-by":"crossref","unstructured":"Zhou G, Su J (2002) Named entity recognition using an hmm-based chunk tagger. In: Proceedings of the 40th annual meeting of the association for computational linguistics, pp 473\u2013480","DOI":"10.3115\/1073083.1073163"},{"key":"7733_CR48","doi-asserted-by":"crossref","unstructured":"Zhou W, Chen M (2021) Learning from noisy labels for entity-centric information extraction. Preprint at arXiv:2104.08656","DOI":"10.18653\/v1\/2021.emnlp-main.437"}],"container-title":["Neural Computing and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-022-07733-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00521-022-07733-0\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-022-07733-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,7]],"date-time":"2023-01-07T06:20:54Z","timestamp":1673072454000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00521-022-07733-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,9,24]]},"references-count":48,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2023,1]]}},"alternative-id":["7733"],"URL":"https:\/\/doi.org\/10.1007\/s00521-022-07733-0","relation":{},"ISSN":["0941-0643","1433-3058"],"issn-type":[{"value":"0941-0643","type":"print"},{"value":"1433-3058","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,9,24]]},"assertion":[{"value":"27 December 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 August 2022","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"24 September 2022","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}]}}