{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,27]],"date-time":"2026-04-27T05:47:02Z","timestamp":1777268822016,"version":"3.51.4"},"publisher-location":"Cham","reference-count":21,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031971433","type":"print"},{"value":"9783031971440","type":"electronic"}],"license":[{"start":{"date-parts":[[2025,7,1]],"date-time":"2025-07-01T00:00:00Z","timestamp":1751328000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,7,1]],"date-time":"2025-07-01T00:00:00Z","timestamp":1751328000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-3-031-97144-0_13","type":"book-chapter","created":{"date-parts":[[2025,7,1]],"date-time":"2025-07-01T06:09:33Z","timestamp":1751350173000},"page":"140-151","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Repurposing Annotation Guidelines to\u00a0Instruct LLM Annotators: A Case Study"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6970-5370","authenticated-orcid":false,"given":"Kon Woo","family":"Kim","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5651-1860","authenticated-orcid":false,"given":"Rezarta","family":"Islamaj","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8877-3248","authenticated-orcid":false,"given":"Jin-Dong","family":"Kim","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5849-2261","authenticated-orcid":false,"given":"Florian","family":"Boudin","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6544-5076","authenticated-orcid":false,"given":"Akiko","family":"Aizawa","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2025,7,1]]},"reference":[{"issue":"1","key":"13_CR1","doi-asserted-by":"publisher","DOI":"10.2196\/59680","volume":"12","author":"G Andrade","year":"2024","unstructured":"Andrade, G., Yada, S., Aramaki, E., et al.: Is boundary annotation necessary? Evaluating boundary-free approaches to improve clinical named entity annotation efficiency: case study. JMIR Med. Inf. 12(1), e59680 (2024). https:\/\/doi.org\/10.2196\/59680","journal-title":"JMIR Med. Inf."},{"key":"13_CR2","doi-asserted-by":"publisher","unstructured":"Beltagy, I., Lo, K., Cohan, A.: SciBERT: a pretrained language model for scientific text. In: Inui, K., Jiang, J., Ng, V., Wan, X. (eds.) Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), pp. 3615\u20133620. Association for Computational Linguistics, Hong Kong (2019). https:\/\/doi.org\/10.18653\/v1\/D19-1371","DOI":"10.18653\/v1\/D19-1371"},{"key":"13_CR3","doi-asserted-by":"publisher","unstructured":"Callison-Burch, C.: Fast, cheap, and creative: evaluating translation quality using Amazon\u2019s Mechanical Turk. In: Proceedings of the 2009 Conference on Empirical Methods in Natural Language Processing, pp. 286\u2013295 (2009). https:\/\/doi.org\/10.3115\/1699510.1699548","DOI":"10.3115\/1699510.1699548"},{"key":"13_CR4","doi-asserted-by":"publisher","unstructured":"Ding, B., et al.: Is GPT-3 a good data annotator? In: Rogers, A., Boyd-Graber, J., Okazaki, N. (eds.) Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics, vol. 1: Long Papers, pp. 11173\u201311195. Association for Computational Linguistics, Toronto (2023). https:\/\/doi.org\/10.18653\/v1\/2023.acl-long.626","DOI":"10.18653\/v1\/2023.acl-long.626"},{"key":"13_CR5","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.jbi.2013.12.006","volume":"47","author":"RI Do\u011fan","year":"2014","unstructured":"Do\u011fan, R.I., Leaman, R., Lu, Z.: NCBI disease corpus: a resource for disease name recognition and concept normalization. J. Biomed. Inf. 47, 1\u201310 (2014). https:\/\/doi.org\/10.1016\/j.jbi.2013.12.006","journal-title":"J. Biomed. Inf."},{"issue":"30","key":"13_CR6","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2305016120","volume":"120","author":"F Gilardi","year":"2023","unstructured":"Gilardi, F., Alizadeh, M., Kubli, M.: ChatGPT outperforms crowd workers for text-annotation tasks. Proc. Natl. Acad. Sci. 120(30), e2305016120 (2023). https:\/\/doi.org\/10.1073\/pnas.2305016120","journal-title":"Proc. Natl. Acad. Sci."},{"key":"13_CR7","unstructured":"Goel, A., et\u00a0al.: LLMs accelerate annotation for medical information extraction. In: Machine Learning for Health (ML4H), pp. 82\u2013100. PMLR (2023)"},{"key":"13_CR8","doi-asserted-by":"publisher","unstructured":"Hirschman, L., Fort, K., Bou\u00e9, S., Kyrpides, N., Islamaj\u00a0Do\u011fan, R., Cohen, K.B.: Crowdsourcing and curation: perspectives from biology and natural language processing. Database 2016, baw115 (2016). https:\/\/doi.org\/10.1093\/database\/baw115","DOI":"10.1093\/database\/baw115"},{"issue":"W1","key":"13_CR9","doi-asserted-by":"publisher","first-page":"W5","DOI":"10.1093\/nar\/gkaa333","volume":"48","author":"R Islamaj","year":"2020","unstructured":"Islamaj, R., Kwon, D., Kim, S., Lu, Z.: TeamTat: a collaborative text annotation tool. Nucleic Acids Res. 48(W1), W5\u2013W11 (2020). https:\/\/doi.org\/10.1093\/nar\/gkaa333","journal-title":"Nucleic Acids Res."},{"issue":"1","key":"13_CR10","doi-asserted-by":"publisher","first-page":"91","DOI":"10.1038\/s41597-021-00875-1","volume":"8","author":"R Islamaj","year":"2021","unstructured":"Islamaj, R., et al.: NLM-Chem, a new resource for chemical entity recognition in pubmed full text literature. Sci. Data 8(1), 91 (2021). https:\/\/doi.org\/10.1038\/s41597-021-00875-1","journal-title":"Sci. Data"},{"key":"13_CR11","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbi.2021.103779","volume":"118","author":"R Islamaj","year":"2021","unstructured":"Islamaj, R., et al.: NLM-Gene, a richly annotated gold standard dataset for gene entities that addresses ambiguity and multi-species gene recognition. J. Biomed. Inf. 118, 103779 (2021). https:\/\/doi.org\/10.1016\/j.jbi.2021.103779","journal-title":"J. Biomed. Inf."},{"issue":"4","key":"13_CR12","doi-asserted-by":"publisher","first-page":"1234","DOI":"10.1093\/bioinformatics\/btz682","volume":"36","author":"J Lee","year":"2020","unstructured":"Lee, J., et al.: BioBERT: a pre-trained biomedical language representation model for biomedical text mining. Bioinformatics 36(4), 1234\u20131240 (2020). https:\/\/doi.org\/10.1093\/bioinformatics\/btz682","journal-title":"Bioinformatics"},{"key":"13_CR13","doi-asserted-by":"publisher","unstructured":"Li, J.: A comparative study on annotation quality of crowdsourcing and LLM via label aggregation. In: ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), pp. 6525\u20136529 (2024). https:\/\/doi.org\/10.1109\/ICASSP48485.2024.10447803","DOI":"10.1109\/ICASSP48485.2024.10447803"},{"key":"13_CR14","doi-asserted-by":"publisher","unstructured":"Li, M., et al.: CoAnnotating: uncertainty-guided work allocation between human and large language models for data annotation. In: Bouamor, H., Pino, J., Bali, K. (eds.) Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, pp. 1487\u20131505. Association for Computational Linguistics, Singapore (2023). https:\/\/doi.org\/10.18653\/v1\/2023.emnlp-main.92","DOI":"10.18653\/v1\/2023.emnlp-main.92"},{"issue":"8","key":"13_CR15","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0307741","volume":"19","author":"CR Liyanage","year":"2024","unstructured":"Liyanage, C.R., Gokani, R., Mago, V.: GPT-4 as an x data annotator: unraveling its performance on a stance classification task. PLoS ONE 19(8), e0307741 (2024). https:\/\/doi.org\/10.1371\/journal.pone.0307741","journal-title":"PLoS ONE"},{"key":"13_CR16","doi-asserted-by":"publisher","unstructured":"Luo, L., Lai, P.T., Wei, C.H., Arighi, C.N., Lu, Z.: BioRED: a rich biomedical relation extraction dataset. Brief. Bioinf. 23(5), bbac282 (2022). https:\/\/doi.org\/10.1093\/bib\/bbac282","DOI":"10.1093\/bib\/bbac282"},{"key":"13_CR17","doi-asserted-by":"publisher","unstructured":"Sainz, O., Garc\u00eda-Ferrero, I., Agerri, R., de\u00a0Lacalle, O.L., Rigau, G., Agirre, E.: GoLLIE: annotation guidelines improve zero-shot information-extraction. CoRR arxiv:2310.03668 (2023). https:\/\/doi.org\/10.48550\/ARXIV.2310.03668","DOI":"10.48550\/ARXIV.2310.03668"},{"key":"13_CR18","doi-asserted-by":"publisher","unstructured":"Snow, R., O\u2019connor, B., Jurafsky, D., Ng, A.Y.: Cheap and fast\u2013but is it good? Evaluating non-expert annotations for natural language tasks. In: Proceedings of the 2008 Conference on Empirical Methods in Natural Language Processing, pp. 254\u2013263 (2008). https:\/\/doi.org\/10.3115\/1613715.1613751","DOI":"10.3115\/1613715.1613751"},{"key":"13_CR19","doi-asserted-by":"publisher","unstructured":"Wang, S., Liu, Y., Xu, Y., Zhu, C., Zeng, M.: Want to reduce labeling cost? GPT-3 can help. In: Moens, M.F., Huang, X., Specia, L., Yih, S.W.T. (eds.) Findings of the Association for Computational Linguistics: EMNLP 2021, pp. 4195\u20134205. Association for Computational Linguistics, Punta Cana (2021). DOI: https:\/\/doi.org\/10.18653\/v1\/2021.findings-emnlp.354","DOI":"10.18653\/v1\/2021.findings-emnlp.354"},{"issue":"W1","key":"13_CR20","doi-asserted-by":"publisher","first-page":"W518","DOI":"10.1093\/nar\/gkt441","volume":"41","author":"CH Wei","year":"2013","unstructured":"Wei, C.H., Kao, H.Y., Lu, Z.: PubTator: a web-based text mining tool for assisting biocuration. Nucleic Acids Res. 41(W1), W518\u2013W522 (2013). https:\/\/doi.org\/10.1093\/nar\/gkt441","journal-title":"Nucleic Acids Res."},{"key":"13_CR21","doi-asserted-by":"publisher","unstructured":"Zhu, Y., Zhang, P., ul\u00a0Haq, E., Hui, P., Tyson, G.: Can ChatGPT reproduce human-generated labels? a study of social computing tasks. CoRR arxiv:2304.10145 (2023). https:\/\/doi.org\/10.48550\/ARXIV.2304.10145","DOI":"10.48550\/ARXIV.2304.10145"}],"container-title":["Lecture Notes in Computer Science","Natural Language Processing and Information Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-97144-0_13","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,27]],"date-time":"2026-04-27T05:18:37Z","timestamp":1777267117000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-97144-0_13"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,1]]},"ISBN":["9783031971433","9783031971440"],"references-count":21,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-97144-0_13","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,7,1]]},"assertion":[{"value":"1 July 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors have no competing interests to declare that are relevant to the content of this article.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"NLDB","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Applications of Natural Language to Information Systems","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Kanazawa","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Japan","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2025","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"4 July 2025","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"6 July 2025","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"30","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"nldb2025","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/nldb2025.github.io\/index.html","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}