{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,28]],"date-time":"2025-06-28T07:24:20Z","timestamp":1751095460954,"version":"3.40.3"},"publisher-location":"Singapore","reference-count":26,"publisher":"Springer Nature Singapore","isbn-type":[{"type":"print","value":"9789819783663"},{"type":"electronic","value":"9789819783670"}],"license":[{"start":{"date-parts":[[2024,11,29]],"date-time":"2024-11-29T00:00:00Z","timestamp":1732838400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,11,29]],"date-time":"2024-11-29T00:00:00Z","timestamp":1732838400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-981-97-8367-0_22","type":"book-chapter","created":{"date-parts":[[2024,11,28]],"date-time":"2024-11-28T11:55:20Z","timestamp":1732794920000},"page":"362-386","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Cost-Efficient Crowdsourcing for\u00a0Span-Based Sequence Labeling: Worker Selection and\u00a0Data Augmentation"],"prefix":"10.1007","author":[{"given":"Yujie","family":"Wang","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Chao","family":"Huang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Liner","family":"Yang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhixuan","family":"Fang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yaping","family":"Huang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yang","family":"Liu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jingsi","family":"Yu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Erhong","family":"Yang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,11,29]]},"reference":[{"key":"22_CR1","unstructured":"Biswas, A., Jain, S., Mandal, D., Narahari, Y.: A truthful budget feasible multi-armed bandit mechanism for crowdsourcing time critical tasks. In: Proceedings of the 2015 International Conference on Autonomous Agents and Multiagent Systems, pp. 1101\u20131109 (2015). http:\/\/dblp.uni-trier.de\/db\/conf\/atal\/aamas2015.html"},{"key":"22_CR2","unstructured":"Casper, S., et\u00a0al.: Open problems and fundamental limitations of reinforcement learning from human feedback. arXiv preprint arXiv:2307.15217 (2023). https:\/\/arxiv.org\/abs\/2307.15217"},{"key":"22_CR3","unstructured":"Chen, W., Wang, Y., Yuan, Y.: Combinatorial multi-armed bandit: general framework and applications. In: Proceedings of the 30th International Conference on Machine Learning, pp. 151\u2013159 (2013). https:\/\/proceedings.mlr.press\/v28\/chen13a.html"},{"key":"22_CR4","unstructured":"Collobert, R., Weston, J., Bottou, L., Karlen, M., Kavukcuoglu, K., Kuksa, P.: Natural language processing (almost) from scratch. J. Mach. Learn. Res. 12(76), 2493\u20132537 (2011). http:\/\/jmlr.org\/papers\/v12\/collobert11a.html"},{"key":"22_CR5","unstructured":"Derczynski, L.: Complementarity, F-score, and NLP evaluation. In: Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC\u201916), pp. 261\u2013266. European Language Resources Association (ELRA), Portoro\u017e, Slovenia (2016). https:\/\/aclanthology.org\/L16-1040"},{"key":"22_CR6","unstructured":"Erdogan, H.: Sequence Labeling: generative and discriminative approaches. In: Proceedings of the 9th International Conference on Machine Learning and Applications. pp. 1\u2013132 (2010). https:\/\/www.icmla-conference.org\/icmla10\/"},{"key":"22_CR7","unstructured":"Garcelon, E., Avadhanula, V., Lazaric, A., Pirotta, M.: Top k ranking for multi-armed bandit with noisy evaluations. In: Proceedings of The 25th International Conference on Artificial Intelligence and Statistics, pp. 6242\u20136269 (2022). https:\/\/proceedings.mlr.press\/v151\/garcelon22b.html"},{"key":"22_CR8","unstructured":"Howe, J.: The rise of crowdsourcing. Wired Mag. 14(6), 1\u20134 (2006). http:\/\/www.wired.com\/wired\/archive\/14.06\/crowds.html"},{"issue":"1","key":"22_CR9","doi-asserted-by":"publisher","first-page":"4","DOI":"10.1016\/0196-8858(85)90002-8","volume":"6","author":"TL Lai","year":"1985","unstructured":"Lai, T.L., Robbins, H.: Asymptotically efficient adaptive allocation rules. Adv. Appl. Math. 6(1), 4\u201322 (1985). https:\/\/doi.org\/10.1016\/0196-8858(85)90002-8","journal-title":"Adv. Appl. Math."},{"key":"22_CR10","unstructured":"Lewis, D.D., Yang, Y., Rose, T.G., Li, F.: RCV1: a new benchmark collection for text categorization research. J. Mach. Learn. Res. 5, 361\u2013397 (2004). http:\/\/jmlr.org\/papers\/volume5\/lewis04a\/lewis04a.pdf"},{"issue":"5","key":"22_CR11","doi-asserted-by":"publisher","first-page":"2231","DOI":"10.1109\/TNET.2022.3169180","volume":"30","author":"F Li","year":"2022","unstructured":"Li, F., Zhao, J., Yu, D., Cheng, X., Lv, W.: Harnessing context for budget-limited crowdsensing with massive uncertain workers. IEEE\/ACM Trans. Netw. 30(5), 2231\u20132245 (2022). https:\/\/doi.org\/10.1109\/TNET.2022.3169180","journal-title":"IEEE\/ACM Trans. Netw."},{"key":"22_CR12","doi-asserted-by":"publisher","unstructured":"Motwani, R., Raghavan, P.: Randomized Algorithms (1995). https:\/\/doi.org\/10.1017\/CBO9780511814075","DOI":"10.1017\/CBO9780511814075"},{"key":"22_CR13","doi-asserted-by":"publisher","unstructured":"Nangia, N., Sugawara, S., Trivedi, H., Warstadt, A., Vania, C., Bowman, S.R.: What ingredients make for an effective crowdsourcing protocol for difficult NLU data collection tasks? In: Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers), pp. 1221\u20131235. Association for Computational Linguistics, Online (2021). https:\/\/doi.org\/10.18653\/v1\/2021.acl-long.98, https:\/\/aclanthology.org\/2021.acl-long.98","DOI":"10.18653\/v1\/2021.acl-long.98"},{"key":"22_CR14","doi-asserted-by":"publisher","unstructured":"Nguyen, A.T., Wallace, B., Li, J.J., Nenkova, A., Lease, M.: Aggregating and predicting sequence labels from crowd annotations. In: Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 299\u2013309. Association for Computational Linguistics, Vancouver, Canada (2017). https:\/\/doi.org\/10.18653\/v1\/P17-1028, https:\/\/aclanthology.org\/P17-1028","DOI":"10.18653\/v1\/P17-1028"},{"key":"22_CR15","doi-asserted-by":"publisher","unstructured":"Nowak, S., R\u00fcger, S.: How reliable are annotations via crowdsourcing: a study about inter-annotator agreement for multi-label image annotation. In: Proceedings of the International Conference on Multimedia Information Retrieval, pp. 557\u2013566 (2010). https:\/\/doi.org\/10.1145\/1743384.1743478","DOI":"10.1145\/1743384.1743478"},{"key":"22_CR16","unstructured":"Ramshaw, L., Marcus, M.: Text chunking using transformation-based learning. In: Third Workshop on Very Large Corpora (1995). https:\/\/aclanthology.org\/W95-0107"},{"key":"22_CR17","unstructured":"Rangi, A., Franceschetti, M.: Multi-armed bandit algorithms for crowdsourcing systems with online estimation of workers\u2019 ability. In: Proceedings of the 17th International Conference on Autonomous Agents and MultiAgent Systems, pp. 1345\u20131352 (2018). http:\/\/dl.acm.org\/citation.cfm?id=3237900"},{"issue":"2","key":"22_CR18","doi-asserted-by":"publisher","first-page":"165","DOI":"10.1007\/s10994-013-5411-2","volume":"95","author":"F Rodrigues","year":"2014","unstructured":"Rodrigues, F., Pereira, F., Ribeiro, B.: Sequence labeling with multiple annotators. Mach. Learn. 95(2), 165\u2013181 (2014). https:\/\/doi.org\/10.1007\/s10994-013-5411-2","journal-title":"Mach. Learn."},{"key":"22_CR19","doi-asserted-by":"publisher","unstructured":"Rodrigues, F., Pereira, F.C.: Deep learning from crowds. In: Proceedings of the Thirty-Second AAAI Conference on Artificial Intelligence and Thirtieth Innovative Applications of Artificial Intelligence Conference and Eighth AAAI Symposium on Educational Advances in Artificial Intelligence, pp. 1611\u20131618 (2018). https:\/\/doi.org\/10.1609\/aaai.v32i1.11506","DOI":"10.1609\/aaai.v32i1.11506"},{"key":"22_CR20","doi-asserted-by":"publisher","unstructured":"Simpson, E., Gurevych, I.: A Bayesian approach for sequence tagging with crowds. In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), pp. 1093\u20131104. Association for Computational Linguistics, Hong Kong, China (2019). https:\/\/doi.org\/10.18653\/v1\/D19-1101, https:\/\/aclanthology.org\/D19-1101","DOI":"10.18653\/v1\/D19-1101"},{"key":"22_CR21","doi-asserted-by":"publisher","unstructured":"Song, Y., Jin, H.: Minimizing entropy for crowdsourcing with combinatorial multi-armed bandit. IEEE Conf. Comput. Commun., 1\u201310 (2021). https:\/\/doi.org\/10.1109\/INFOCOM42981.2021.9488800","DOI":"10.1109\/INFOCOM42981.2021.9488800"},{"key":"22_CR22","doi-asserted-by":"crossref","unstructured":"Tjong Kim\u00a0Sang, E.F., De\u00a0Meulder, F.: Introduction to the CoNLL-2003 shared task: language-independent named entity recognition. In: Proceedings of the Seventh Conference on Natural Language Learning at HLT-NAACL 2003, pp. 142\u2013147 (2003). https:\/\/aclanthology.org\/W03-0419","DOI":"10.3115\/1119176.1119195"},{"key":"22_CR23","doi-asserted-by":"publisher","first-page":"89","DOI":"10.1016\/j.artint.2014.04.005","volume":"214","author":"L Tran-Thanh","year":"2014","unstructured":"Tran-Thanh, L., Stein, S., Rogers, A., Jennings, N.R.: Efficient crowdsourcing of unknown experts using bounded multi-armed bandits. Artif. Intell. 214, 89\u2013111 (2014). https:\/\/doi.org\/10.1016\/j.artint.2014.04.005","journal-title":"Artif. Intell."},{"key":"22_CR24","doi-asserted-by":"publisher","unstructured":"Venanzi, M., Guiver, J., Kazai, G., Kohli, P., Shokouhi, M.: Community-based Bayesian aggregation models for crowdsourcing. In: Proceedings of the 23rd International Conference on World Wide Web, pp. 155\u2013164 (2014). https:\/\/doi.org\/10.1145\/2566486.2567989","DOI":"10.1145\/2566486.2567989"},{"key":"22_CR25","doi-asserted-by":"publisher","unstructured":"Zhang, X., Xu, G., Sun, Y., Zhang, M., Wang, X., Zhang, M.: Identifying Chinese opinion expressions with extremely-noisy crowdsourcing annotations. In: Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 2801\u20132813. Association for Computational Linguistics, Dublin, Ireland (2022). https:\/\/doi.org\/10.18653\/v1\/2022.acl-long.200, https:\/\/aclanthology.org\/2022.acl-long.200","DOI":"10.18653\/v1\/2022.acl-long.200"},{"key":"22_CR26","doi-asserted-by":"publisher","unstructured":"\u0130ren, D., Bilgen, S.: Cost of quality in crowdsourcing. Hum. Comput. 1 (2014). https:\/\/doi.org\/10.15346\/hc.v1i2.14","DOI":"10.15346\/hc.v1i2.14"}],"container-title":["Lecture Notes in Computer Science","Chinese Computational Linguistics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-981-97-8367-0_22","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,28]],"date-time":"2024-11-28T12:08:07Z","timestamp":1732795687000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-981-97-8367-0_22"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,11,29]]},"ISBN":["9789819783663","9789819783670"],"references-count":26,"URL":"https:\/\/doi.org\/10.1007\/978-981-97-8367-0_22","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2024,11,29]]},"assertion":[{"value":"29 November 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors have no competing interests to declare that are relevant to the content of this article.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"CCL","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"China National Conference on Chinese Computational Linguistics","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Taiyuan","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"China","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"25 July 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"28 July 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"23","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"cncl2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/cips-cl.org\/static\/CCL2024\/en\/index.html","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}