{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,14]],"date-time":"2026-05-14T20:26:12Z","timestamp":1778790372021,"version":"3.51.4"},"reference-count":32,"publisher":"Oxford University Press (OUP)","issue":"W1","funder":[{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Ministry of Science and ICT","award":["NRF-2014M3C9A3064706"],"award-info":[{"award-number":["NRF-2014M3C9A3064706"]}]},{"DOI":"10.13039\/100009950","name":"Ministry of Education","doi-asserted-by":"publisher","award":["NRF-2018R1D1A1B07044775"],"award-info":[{"award-number":["NRF-2018R1D1A1B07044775"]}],"id":[{"id":"10.13039\/100009950","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020,7,2]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:p>Manually annotated data is key to developing text-mining and information-extraction algorithms. However, human annotation requires considerable time, effort and expertise. Given the rapid growth of biomedical literature, it is paramount to build tools that facilitate speed and maintain expert quality. While existing text annotation tools may provide user-friendly interfaces to domain experts, limited support is available for figure display, project management, and multi-user team annotation. In response, we developed TeamTat (https:\/\/www.teamtat.org), a web-based annotation tool (local setup available), equipped to manage team annotation projects engagingly and efficiently. TeamTat is a novel tool for managing multi-user, multi-label document annotation, reflecting the entire production life cycle. Project managers can specify annotation schema for entities and relations and select annotator(s) and distribute documents anonymously to prevent bias. Document input format can be plain text, PDF or BioC\u00a0(uploaded locally or automatically retrieved from PubMed\/PMC), and output format is BioC with inline annotations. TeamTat displays figures from the full text for the annotator's convenience. Multiple users can work on the same document independently in their workspaces, and the team manager can track task completion. TeamTat provides corpus quality assessment via inter-annotator agreement statistics, and a user-friendly interface convenient for annotation review and inter-annotator disagreement resolution to improve corpus quality.<\/jats:p>","DOI":"10.1093\/nar\/gkaa333","type":"journal-article","created":{"date-parts":[[2020,4,22]],"date-time":"2020-04-22T19:09:43Z","timestamp":1587582583000},"page":"W5-W11","source":"Crossref","is-referenced-by-count":63,"title":["TeamTat: a collaborative text annotation tool"],"prefix":"10.1093","volume":"48","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5651-1860","authenticated-orcid":false,"given":"Rezarta","family":"Islamaj","sequence":"first","affiliation":[{"name":"National Center for Biotechnology Information (NCBI), National Library of Medicine (NLM), National Institutes of Health (NIH), Bethesda, MD 20894, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dongseop","family":"Kwon","sequence":"first","affiliation":[{"name":"School of Software Convergence, Myongji University, Seoul 03674, South Korea"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3072-6649","authenticated-orcid":false,"given":"Sun","family":"Kim","sequence":"first","affiliation":[{"name":"National Center for Biotechnology Information (NCBI), National Library of Medicine (NLM), National Institutes of Health (NIH), Bethesda, MD 20894, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9998-916X","authenticated-orcid":false,"given":"Zhiyong","family":"Lu","sequence":"first","affiliation":[{"name":"National Center for Biotechnology Information (NCBI), National Library of Medicine (NLM), National Institutes of Health (NIH), Bethesda, MD 20894, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"286","published-online":{"date-parts":[[2020,5,8]]},"reference":[{"key":"2020062614040639400_B1","doi-asserted-by":"crossref","first-page":"baw147","DOI":"10.1093\/database\/baw147","article-title":"The BioC-BioGRID corpus: full text articles annotated for curation of protein-protein and genetic interactions","volume":"2017","author":"Islamaj\u00a0Dogan","year":"2017","journal-title":"Database"},{"key":"2020062614040639400_B2","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/j.jbi.2013.12.006","article-title":"NCBI disease corpus: a resource for disease name recognition and concept normalization","volume":"47","author":"Dogan","year":"2014","journal-title":"J. Biomed. Inform."},{"key":"2020062614040639400_B3","doi-asserted-by":"crossref","first-page":"S2","DOI":"10.1186\/1758-2946-7-S1-S2","article-title":"The CHEMDNER corpus of chemicals and drugs and its annotation principles","volume":"7","author":"Krallinger","year":"2015","journal-title":"J. Cheminform."},{"key":"2020062614040639400_B4","doi-asserted-by":"crossref","first-page":"baw068","DOI":"10.1093\/database\/baw068","article-title":"BioCreative V CDR task corpus: a resource for chemical disease relation extraction","volume":"2016","author":"Li","year":"2016","journal-title":"Database"},{"key":"2020062614040639400_B5","doi-asserted-by":"crossref","first-page":"bau074","DOI":"10.1093\/database\/bau074","article-title":"BC4GO: a full-text corpus for the BioCreative IV GO task","volume":"2014","author":"Van\u00a0Auken","year":"2014","journal-title":"Database"},{"key":"2020062614040639400_B6","doi-asserted-by":"crossref","first-page":"1433","DOI":"10.1093\/bioinformatics\/btt156","article-title":"tmVar: a text mining approach for extracting sequence variants in biomedical literature","volume":"29","author":"Wei","year":"2013","journal-title":"Bioinformatics"},{"key":"2020062614040639400_B7","first-page":"16","volume-title":"Proceedings of the Third Workshop on Building and Evaluation Resources for Biomedical Text Mining (BioTxtM 2012) at Language Resources and Evaluation (LREC)","author":"Neves","year":"2012"},{"key":"2020062614040639400_B8","doi-asserted-by":"crossref","first-page":"baw005","DOI":"10.1093\/database\/baw005","article-title":"EXTRACT: interactive extraction of environment metadata and term suggestion for metagenomic sample annotation","volume":"2016","author":"Pafilis","year":"2016","journal-title":"Database"},{"key":"2020062614040639400_B9","doi-asserted-by":"crossref","first-page":"327","DOI":"10.1093\/bib\/bbs084","article-title":"A survey on annotation tools for the biomedical literature","volume":"15","author":"Neves","year":"2014","journal-title":"Brief. Bioinform."},{"key":"2020062614040639400_B10","doi-asserted-by":"crossref","DOI":"10.1093\/bib\/bbz130","article-title":"An extensive review of tools for manual annotation of documents","author":"Neves","year":"2019","journal-title":"Brief. Bioinform."},{"key":"2020062614040639400_B11","doi-asserted-by":"crossref","first-page":"bas010","DOI":"10.1093\/database\/bas010","article-title":"Argo: an integrative, interactive, text mining-based workbench supporting curation","volume":"2012","author":"Rak","year":"2012","journal-title":"Database"},{"key":"2020062614040639400_B12","doi-asserted-by":"crossref","first-page":"bau048","DOI":"10.1093\/database\/bau048","article-title":"Egas: a collaborative and interactive document curation platform","volume":"2014","author":"Campos","year":"2014","journal-title":"Database"},{"key":"2020062614040639400_B13","doi-asserted-by":"crossref","first-page":"139","DOI":"10.1016\/j.cmpb.2013.03.007","article-title":"BioAnnote: a software platform for annotating biomedical documents with application in medical learning environments","volume":"111","author":"Lopez-Fernandez","year":"2013","journal-title":"Comput. Methods Programs Biomed."},{"key":"2020062614040639400_B14","doi-asserted-by":"crossref","first-page":"1007","DOI":"10.1007\/s10579-013-9215-6","article-title":"GATE Teamware: a web-based, collaborative text annotation framework","volume":"47","author":"Bontcheva","year":"2013","journal-title":"Lang. Resour. Eval."},{"key":"2020062614040639400_B15","doi-asserted-by":"crossref","first-page":"242","DOI":"10.1016\/j.cmpb.2014.11.005","article-title":"Marky: a tool supporting annotation consistency in multi-user and iterative document annotation projects","volume":"118","author":"Perez-Perez","year":"2015","journal-title":"Comput. Methods Prog. Biol."},{"key":"2020062614040639400_B16","first-page":"269","article-title":"Marky: a lightweight web tracking tool for document annotation","volume":"294","author":"Perez-Perez","year":"2014","journal-title":"Adv. Intell. Syst."},{"key":"2020062614040639400_B17","doi-asserted-by":"crossref","first-page":"W518","DOI":"10.1093\/nar\/gkt441","article-title":"PubTator: a web-based text mining tool for assisting biocuration","volume":"41","author":"Wei","year":"2013","journal-title":"Nucleic Acids Res."},{"key":"2020062614040639400_B18","doi-asserted-by":"crossref","first-page":"W523","DOI":"10.1093\/nar\/gky428","article-title":"ezTag: tagging biomedical concepts via interactive learning","volume":"46","author":"Kwon","year":"2018","journal-title":"Nucleic Acids Res."},{"key":"2020062614040639400_B19","doi-asserted-by":"crossref","first-page":"1984","DOI":"10.1371\/journal.pbio.0020309","article-title":"Textpresso: an ontology-based information retrieval and extraction system for biological literature","volume":"2","author":"Muller","year":"2004","journal-title":"PLoS Biol."},{"key":"2020062614040639400_B20","doi-asserted-by":"crossref","first-page":"94","DOI":"10.1186\/s12859-018-2103-8","article-title":"Textpresso Central: a customizable platform for searching, text mining, viewing, and curating biomedical literature","volume":"19","author":"Muller","year":"2018","journal-title":"BMC Bioinformatics"},{"key":"2020062614040639400_B21","doi-asserted-by":"crossref","first-page":"bau067","DOI":"10.1093\/database\/bau067","article-title":"Assisting manual literature curation for protein-protein interactions using BioQRator","volume":"2014","author":"Kwon","year":"2014","journal-title":"Database"},{"key":"2020062614040639400_B22","first-page":"102","volume-title":"Proceedings of the Demonstrations at the 13th Conference of the European Chapter of the Association for Computational Linguistics","author":"Stenetorp","year":"2012"},{"key":"2020062614040639400_B23","doi-asserted-by":"crossref","first-page":"D23","DOI":"10.1093\/nar\/gky1069","article-title":"Database resources of the national center for biotechnology information","volume":"47","author":"Sayers","year":"2019","journal-title":"Nucleic Acids Res."},{"key":"2020062614040639400_B24","doi-asserted-by":"crossref","first-page":"bat064","DOI":"10.1093\/database\/bat064","article-title":"BioC: a minimalist approach to interoperability for biomedical text processing","volume":"2013","author":"Comeau","year":"2013","journal-title":"Database"},{"key":"2020062614040639400_B25","doi-asserted-by":"crossref","first-page":"3533","DOI":"10.1093\/bioinformatics\/btz070","article-title":"PMC text mining subset in BioC: about three million full-text articles and growing","volume":"35","author":"Comeau","year":"2019","journal-title":"Bioinformatics"},{"key":"2020062614040639400_B26","first-page":"202","article-title":"PubAnnotation: a persistent and sharable corpus and annotation repository","author":"Kim","year":"2012","journal-title":"Proceedings of the 2012 Workshop on Biomedical Natural Language Processing"},{"key":"2020062614040639400_B27","volume-title":"International Conference on Biomedical Ontology and BioCreative (ICBO BioCreative 2016)","author":"Comeau","year":"2016"},{"key":"2020062614040639400_B28","first-page":"36","volume-title":"BioCreative V Workshop","author":"Do\u011fan","year":"2015"},{"key":"2020062614040639400_B29","doi-asserted-by":"crossref","first-page":"baw121","DOI":"10.1093\/database\/baw121","article-title":"BioCreative V BioC track overview: collaborative biocurator assistant task for BioGRID","volume":"2016","author":"Kim","year":"2016","journal-title":"Database"},{"key":"2020062614040639400_B30","doi-asserted-by":"crossref","DOI":"10.18653\/v1\/W17-2321","article-title":"BioCreative VI Precision Medicine Track: creating a training corpus for mining protein-protein interactions affected by mutations","volume-title":"Proceedings of the 2017 ACL Workshop on Biomedical Natural Language Processing (BioNLP)","author":"Islamaj\u00a0Dogan","year":"2017"},{"key":"2020062614040639400_B31","doi-asserted-by":"crossref","first-page":"W587","DOI":"10.1093\/nar\/gkz389","article-title":"PubTator central: automated concept annotation for biomedical full text articles","volume":"47","author":"Wei","year":"2019","journal-title":"Nucleic Acids Res."},{"key":"2020062614040639400_B32","doi-asserted-by":"crossref","first-page":"baw120","DOI":"10.1093\/database\/baw120","article-title":"The Markyt visualisation, prediction and benchmark platform for chemical and gene entity recognition at BioCreative\/CHEMDNER challenge","volume":"2016","author":"Perez-Perez","year":"2016","journal-title":"Database"}],"container-title":["Nucleic Acids Research"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/academic.oup.com\/nar\/article-pdf\/48\/W1\/W5\/33433452\/gkaa333.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"http:\/\/academic.oup.com\/nar\/article-pdf\/48\/W1\/W5\/33433452\/gkaa333.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,6,27]],"date-time":"2020-06-27T06:25:05Z","timestamp":1593239105000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/nar\/article\/48\/W1\/W5\/5834578"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,5,8]]},"references-count":32,"journal-issue":{"issue":"W1","published-online":{"date-parts":[[2020,5,8]]},"published-print":{"date-parts":[[2020,7,2]]}},"URL":"https:\/\/doi.org\/10.1093\/nar\/gkaa333","relation":{},"ISSN":["0305-1048","1362-4962"],"issn-type":[{"value":"0305-1048","type":"print"},{"value":"1362-4962","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2020,7,2]]},"published":{"date-parts":[[2020,5,8]]}}}