{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,27]],"date-time":"2026-05-27T20:50:19Z","timestamp":1779915019817,"version":"3.53.1"},"reference-count":95,"publisher":"Oxford University Press (OUP)","issue":"1","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Brief Bioinform"],"published-print":{"date-parts":[[2016,1]]},"DOI":"10.1093\/bib\/bbv021","type":"journal-article","created":{"date-parts":[[2015,4,17]],"date-time":"2015-04-17T21:32:37Z","timestamp":1429306357000},"page":"23-32","source":"Crossref","is-referenced-by-count":87,"title":["Crowdsourcing in biomedicine: challenges and opportunities"],"prefix":"10.1093","volume":"17","author":[{"given":"Ritu","family":"Khare","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Benjamin M.","family":"Good","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Robert","family":"Leaman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Andrew I.","family":"Su","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhiyong","family":"Lu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"286","published-online":{"date-parts":[[2015,4,17]]},"reference":[{"key":"2016011923430581000_17.1.23.1","doi-asserted-by":"publisher","DOI":"10.1177\/0165551512437638"},{"key":"2016011923430581000_17.1.23.2","doi-asserted-by":"publisher","DOI":"10.1093\/database\/bap018"},{"key":"2016011923430581000_17.1.23.3","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4939-0709-0_2"},{"key":"2016011923430581000_17.1.23.4","doi-asserted-by":"publisher","DOI":"10.1093\/database\/bav016"},{"key":"2016011923430581000_17.1.23.5","doi-asserted-by":"publisher","DOI":"10.1093\/database\/bau094"},{"key":"2016011923430581000_17.1.23.6","doi-asserted-by":"publisher","DOI":"10.1093\/jnci\/dju258"},{"key":"2016011923430581000_17.1.23.7","doi-asserted-by":"publisher","DOI":"10.1186\/gb-2011-12-12-135"},{"key":"2016011923430581000_17.1.23.8","doi-asserted-by":"publisher","DOI":"10.1136\/amiajnl-2012-000852"},{"key":"2016011923430581000_17.1.23.9","doi-asserted-by":"publisher","DOI":"10.1097\/ACM.0b013e31828f86ef"},{"key":"2016011923430581000_17.1.23.10","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/btt333"},{"key":"2016011923430581000_17.1.23.11","doi-asserted-by":"crossref","unstructured":"Snow R O'Connor B Jurafsky D . Cheap and Fast\u2014But is it Good? Evaluating Non-Expert Annotations for Natural Language Tasks. In: Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics, Honolulu, Hawaii, 2008, pp. 254\u201363.","DOI":"10.3115\/1613715.1613751"},{"key":"2016011923430581000_17.1.23.12","unstructured":"Yetisgen-Yildiz M Solti I Xia F . Preliminary experiments with Amazon's mechanical turk for annotating medical named entities. In: NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon's Mechanical Turk. Association for Computational Linguistics, Los Angeles, CA, 2010, pp. 180\u20133."},{"key":"2016011923430581000_17.1.23.13","doi-asserted-by":"publisher","DOI":"10.1186\/1471-2105-12-S9-S2"},{"key":"2016011923430581000_17.1.23.14","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0055814"},{"key":"2016011923430581000_17.1.23.15","doi-asserted-by":"publisher","DOI":"10.1038\/nbt.2495"},{"key":"2016011923430581000_17.1.23.16","doi-asserted-by":"publisher","DOI":"10.1038\/nature09304"},{"key":"2016011923430581000_17.1.23.17","first-page":"267","article-title":"Session Introduction","volume-title":"Pacific Symposium on Biocomputing","volume":"Vol. 20","author":"Leaman","year":"2015"},{"key":"2016011923430581000_17.1.23.18","doi-asserted-by":"publisher","DOI":"10.1136\/amiajnl-2012-001482"},{"key":"2016011923430581000_17.1.23.19","first-page":"306","article-title":"Analyzing search behavior of healthcare professionals for drug safety surveillance","volume-title":"Pacific Symposium on Biocomputing","volume":"Vol. 20","author":"Odgers","year":"2015"},{"key":"2016011923430581000_17.1.23.20","doi-asserted-by":"publisher","DOI":"10.1007\/s40264-014-0155-x"},{"key":"2016011923430581000_17.1.23.21","unstructured":"Leaman R Wojtulewicz L Sullivan R . Towards internet-age pharmacovigilance: extracting adverse drug reactions from user posts to health-related social networks. In: Proceedings of the 2010 Workshop on Biomedical Natural Language Processing, Association for Computational Linguistics, 2010, pp. 117\u201325."},{"key":"2016011923430581000_17.1.23.22","doi-asserted-by":"crossref","first-page":"125ra131","DOI":"10.1126\/scitranslmed.3003377","article-title":"Data-driven prediction of drug effects and interactions","volume":"4","author":"Tatonetti","year":"2012","journal-title":"Sci Trans Med"},{"key":"2016011923430581000_17.1.23.23","doi-asserted-by":"crossref","unstructured":"Yang CC Yang H Jiang L . Postmarketing drug safety surveillance using publicly available health-consumer-contributed content in social media. ACM Trans Manag Inf Syst 2014;5.","DOI":"10.1145\/2576233"},{"key":"2016011923430581000_17.1.23.24","doi-asserted-by":"crossref","unstructured":"Ryan PB Madigan D Stang PE . Medication-wide association studies. CPT Pharmacometr Syst Pharmacol, 2, e76.","DOI":"10.1038\/psp.2013.52"},{"key":"2016011923430581000_17.1.23.25","first-page":"1464","article-title":"A drug-adverse event extraction algorithm to support pharmacovigilance knowledge mining from PubMed citations","volume":"2011","author":"Wang","year":"2011","journal-title":"AMIA"},{"key":"2016011923430581000_17.1.23.26","doi-asserted-by":"publisher","DOI":"10.1007\/s11325-014-0965-1"},{"key":"2016011923430581000_17.1.23.27","doi-asserted-by":"publisher","DOI":"10.1016\/j.ajem.2014.05.052"},{"key":"2016011923430581000_17.1.23.28","unstructured":"Bichero S . Global Smartphone Installed Base Forecast by Operating System for 88 Countries: 2007 to 2017. WWW document, http:\/\/www.strategyanalytics.com\/default.aspx?mod= reportabstractviewer&a0=7834 ."},{"key":"2016011923430581000_17.1.23.29","unstructured":"Pickard KT Swan M . Big Desire to Share Big Health Data: A Shift in Consumer Attitudes toward Personal Health Information. In: AAAI 2014 Spring Symposia: Big Data Becomes Personal: Knowledge into Meaning , Association for the Advancement of Artificial Intelligence, 2014."},{"key":"2016011923430581000_17.1.23.30","doi-asserted-by":"crossref","first-page":"63","DOI":"10.1109\/MPUL.2013.2289467","article-title":"Innovating openly: researchers and patients turn to crowdsourcing to collaborate on clinical trials, drug discovery, and more","volume":"5","author":"Morton","year":"2014","journal-title":"IEEE Pulse"},{"key":"2016011923430581000_17.1.23.31","doi-asserted-by":"crossref","unstructured":"Turner-McGrievy GM Helander EE Kaipainen K . The use of crowdsourcing for dietary self-monitoring: crowdsourced ratings of food pictures are comparable to ratings by trained observers. J Am Med Inf Assoc 2014, doi: 10.1136\/amiajnl-2014-002636.","DOI":"10.1136\/amiajnl-2014-002636"},{"key":"2016011923430581000_17.1.23.32","doi-asserted-by":"publisher","DOI":"10.1136\/amiajnl-2010-000055"},{"key":"2016011923430581000_17.1.23.33","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-008-9074-8"},{"key":"2016011923430581000_17.1.23.34","doi-asserted-by":"publisher","DOI":"10.1197\/jamia.M2935"},{"key":"2016011923430581000_17.1.23.35","doi-asserted-by":"publisher","DOI":"10.1093\/database\/bau074"},{"key":"2016011923430581000_17.1.23.36","doi-asserted-by":"publisher","DOI":"10.1186\/1471-2105-12-S1-S3"},{"key":"2016011923430581000_17.1.23.37","unstructured":"Leaman R Khare R Lu Z . NCBI at 2013 ShARe\/CLEF eHealth Shared Task: Disorder Normalization in Clinical Notes with DNorm. In: CLEF 2013 Evaluation Labs and Workshop. The CLEF Initiative, Valencia - Spain, pp. 23\u201326, 2013."},{"key":"2016011923430581000_17.1.23.38","doi-asserted-by":"crossref","unstructured":"Neveol A Islamaj Dogan R Lu Z . Semi-automatic semantic annotation of PubMed queries: a study on quality, efficiency, satisfaction. J Biomed Informatics, 44, 310\u201318.","DOI":"10.1016\/j.jbi.2010.11.001"},{"key":"2016011923430581000_17.1.23.39","doi-asserted-by":"publisher","DOI":"10.2196\/jmir.2426"},{"key":"2016011923430581000_17.1.23.40","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/btt474"},{"key":"2016011923430581000_17.1.23.41","doi-asserted-by":"publisher","DOI":"10.1186\/s13321-014-0051-5"},{"key":"2016011923430581000_17.1.23.42","unstructured":"Baumgartner W Lu Z Johnson HL . An integrated approach to concept recognition in biomedical text. In: Proceedings of the Second BioCreative Challenge Evaluation Workshop, 2007, Vol. 23, pp. 257\u201371."},{"key":"2016011923430581000_17.1.23.43","doi-asserted-by":"publisher","DOI":"10.1136\/amiajnl-2012-001110"},{"key":"2016011923430581000_17.1.23.44","unstructured":"Lafferty JD McCallum A Pereira FCN . Conditional Random Fields: Probabilistic Models for Segmenting and Labeling Sequence Data. In: Proceedings of ICML \u201801 Proceedings of the Eighteenth International Conference on Machine Learning, 2001, pp. 282\u20139."},{"key":"2016011923430581000_17.1.23.45","first-page":"282","article-title":"Microtask Crowdsourcing for Disease Mention Annotation in PubMed Abstracts","volume-title":"Pacific Symposium on Biocomputing","volume":"Vol. 20","author":"Good","year":"2015"},{"key":"2016011923430581000_17.1.23.46","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbi.2013.12.006"},{"key":"2016011923430581000_17.1.23.47","unstructured":"Dogan RI Lu Z . An improved corpus of disease mentions in PubMed citations. In: Workshop on Biomedical Natural Language Processing. Association for Computational Linguistics, Montreal, Canada, 2012, pp. 91\u20139."},{"key":"2016011923430581000_17.1.23.48","doi-asserted-by":"publisher","DOI":"10.1093\/nar\/gkt441"},{"key":"2016011923430581000_17.1.23.49","doi-asserted-by":"publisher","DOI":"10.1093\/database\/bas041"},{"key":"2016011923430581000_17.1.23.50","doi-asserted-by":"publisher","DOI":"10.1093\/database\/bas056"},{"key":"2016011923430581000_17.1.23.51","doi-asserted-by":"publisher","DOI":"10.1093\/database\/bas043"},{"key":"2016011923430581000_17.1.23.52","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbi.2014.08.004"},{"key":"2016011923430581000_17.1.23.53","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-31040-9_8"},{"key":"2016011923430581000_17.1.23.54","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0038460"},{"key":"2016011923430581000_17.1.23.55","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/btq667"},{"key":"2016011923430581000_17.1.23.56","doi-asserted-by":"crossref","unstructured":"Mortensen JM Minty EP Januszyk M . Using the wisdom of the crowds to find critical errors in biomedical ontologies: a study of SNOMED CT. J Am Med Inform Assoc 2014, doi: 10.1136\/amiajnl-2014-002901.","DOI":"10.1136\/amiajnl-2014-002901"},{"key":"2016011923430581000_17.1.23.57","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0100647"},{"key":"2016011923430581000_17.1.23.58","doi-asserted-by":"publisher","DOI":"10.1093\/nar\/gkr1195"},{"key":"2016011923430581000_17.1.23.59","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0071171"},{"key":"2016011923430581000_17.1.23.60","doi-asserted-by":"publisher","DOI":"10.1186\/1471-2164-12-603"},{"key":"2016011923430581000_17.1.23.61","first-page":"330","article-title":"Crowdsourcing RNA Structural Alignments with an Online Computer Game","volume-title":"Pacific Symposium on Biocomputing","volume":"Vol. 20","author":"Waldisp\u00fchl","year":"2015"},{"key":"2016011923430581000_17.1.23.62","doi-asserted-by":"crossref","first-page":"58","DOI":"10.1145\/1378704.1378719","article-title":"Designing games with a purpose","volume":"51","author":"Ahn","year":"2008","journal-title":"Commun ACM"},{"key":"2016011923430581000_17.1.23.63","unstructured":"Gottl F . Crowdsourcing with Gaminification. Advances in Embedded Interactive Systems, Vol. 2, Passau , Germany, 2014."},{"key":"2016011923430581000_17.1.23.64","doi-asserted-by":"publisher","DOI":"10.1039\/C4FD00014E"},{"key":"2016011923430581000_17.1.23.65","doi-asserted-by":"crossref","unstructured":"Perry D Aragon C Cruz S . Human centered game design for bioinformatics and cyberinfrastructure learning. In: Proceedings of the ACM Conference on Extreme Science and Engineering Discovery Environment: Gateway to Discovery. Association for Computing Machinery (ACM), 2013.","DOI":"10.1145\/2484762.2484833"},{"key":"2016011923430581000_17.1.23.66","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0031362"},{"key":"2016011923430581000_17.1.23.67","doi-asserted-by":"publisher","DOI":"10.1038\/nsmb.2119"},{"key":"2016011923430581000_17.1.23.68","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1313039111"},{"key":"2016011923430581000_17.1.23.69","doi-asserted-by":"publisher","DOI":"10.1016\/j.tibs.2014.08.005"},{"key":"2016011923430581000_17.1.23.70","doi-asserted-by":"publisher","DOI":"10.1039\/C0LC00399A"},{"key":"2016011923430581000_17.1.23.71","unstructured":"Riedel-Kruse I Blikstein P . Biotic games and cloud experimentation as novel media for biophysics education. Bulletin of the American Physical Society 2014;59."},{"key":"2016011923430581000_17.1.23.72","doi-asserted-by":"publisher","DOI":"10.1002\/prot.340230303"},{"key":"2016011923430581000_17.1.23.73","doi-asserted-by":"publisher","DOI":"10.1093\/database\/bau039"},{"key":"2016011923430581000_17.1.23.74","doi-asserted-by":"publisher","DOI":"10.1093\/database\/bas049"},{"key":"2016011923430581000_17.1.23.75","doi-asserted-by":"crossref","unstructured":"Kim JD Pyysalo S Ohta T . Overview of BioNLP Shared Task 2011. In: Proceedings of BioNLP Shared Task 2011 Workshop. Association for Computational Linguistics, Portland, OR, 2011, pp. 1\u20136.","DOI":"10.3115\/1572340.1572342"},{"key":"2016011923430581000_17.1.23.76","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-008-9076-6"},{"key":"2016011923430581000_17.1.23.77","doi-asserted-by":"publisher","DOI":"10.1136\/amiajnl-2011-000203"},{"key":"2016011923430581000_17.1.23.78","doi-asserted-by":"crossref","unstructured":"Huang M Lu Z . Community Challenges in Biomedical Text Mining over 10 Years: Success, Failure, and the Future. Brief Bioinformatics 2015, doi: 10.1093\/bib\/bbv024.","DOI":"10.1093\/bib\/bbv024"},{"key":"2016011923430581000_17.1.23.79","doi-asserted-by":"publisher","DOI":"10.1038\/nmeth.2016"},{"key":"2016011923430581000_17.1.23.80","doi-asserted-by":"crossref","first-page":"181re181","DOI":"10.1126\/scitranslmed.3006112","article-title":"Systematic analysis of challenge-driven improvements in molecular prognostic models for breast cancer","volume":"5","author":"Margolin","year":"2013","journal-title":"Sci Trans Med"},{"key":"2016011923430581000_17.1.23.81","doi-asserted-by":"publisher","DOI":"10.1002\/prot.24538"},{"key":"2016011923430581000_17.1.23.82","unstructured":"Ross J Irani I Silberman MS . Who are the Crowdworkers?: Shifting Demographics in Amazon Mechanical Turk. In: CHI \u201810 Extended Abstracts on Human Factors in Computing Systems. ACM, Atlanta, Georgia, 2010, pp. 2863\u201372."},{"key":"2016011923430581000_17.1.23.83","doi-asserted-by":"publisher","DOI":"10.1038\/494155a"},{"key":"2016011923430581000_17.1.23.84","doi-asserted-by":"crossref","unstructured":"Fort K Adda G Cohen KB . Amazon Mechanical Turk: Gold Mine or Coal Mine?. In: Computational Linguistics, 37(2) (2011), pp. 413\u201320.","DOI":"10.1162\/COLI_a_00057"},{"key":"2016011923430581000_17.1.23.85","unstructured":"Machine D Ophoff J . Understanding What Motivates Participation on Crowdsourcing Platforms. In: e-Skills for Knowledge Production and Innovation Conference, Cape Town, South Africa, 2014, pp. 191\u2013200."},{"key":"2016011923430581000_17.1.23.86","doi-asserted-by":"publisher","DOI":"10.1177\/1524839912470654"},{"key":"2016011923430581000_17.1.23.87","doi-asserted-by":"crossref","unstructured":"Khare R An Y Song IY . Can clinicians create high-quality databases: a study on a flexible electronic health record (fEHR) system. In: International Health Informatics Symposium. ACM, Washington, DC, 2010, pp. 8\u201317.","DOI":"10.1145\/1882992.1882997"},{"key":"2016011923430581000_17.1.23.88","doi-asserted-by":"publisher","DOI":"10.1136\/amiajnl-2013-001613"},{"key":"2016011923430581000_17.1.23.89","unstructured":"Khare R An Y Wolf S . Understanding the EMR error control practices among gynecologic physicians. In: iConference 2013, iSchools, Fort Worth, Texas, 2013, pp. 289\u2013301."},{"key":"2016011923430581000_17.1.23.90","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pcbi.1003799"},{"key":"2016011923430581000_17.1.23.91","first-page":"e108","article-title":"Crowdsourcing participatory evaluation of medical pictograms using Amazon Mechanical Turk","volume":"15","author":"Yu","year":"2013","journal-title":"J Med Int Res"},{"key":"2016011923430581000_17.1.23.92","doi-asserted-by":"publisher","DOI":"10.1136\/postgradmedj-2013-132486"},{"key":"2016011923430581000_17.1.23.93","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0100662"},{"key":"2016011923430581000_17.1.23.94","doi-asserted-by":"crossref","unstructured":"Harpaz R DuMouchel W LePendu P . Empirical Bayes Model to Combine Signals of Adverse Drug Reactions. In: Proceedings of the 19th ACM SIGKDD international conference on Knowledge discovery and data mining, 2013, pp. 1339\u201347.","DOI":"10.1145\/2487575.2488214"},{"key":"2016011923430581000_17.1.23.95","unstructured":"Neilson J . Participation inequality: lurkers vs. contributors in internet communities. Jakob Nielsen\u2019s Alert-box , 2006."}],"container-title":["Briefings in Bioinformatics"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/academic.oup.com\/bib\/article-pdf\/17\/1\/23\/6684984\/bbv021.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,5,22]],"date-time":"2025-05-22T14:24:00Z","timestamp":1747923840000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/bib\/article-lookup\/doi\/10.1093\/bib\/bbv021"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,4,17]]},"references-count":95,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2016,1,19]]},"published-print":{"date-parts":[[2016,1]]}},"alternative-id":["10.1093\/bib\/bbv021"],"URL":"https:\/\/doi.org\/10.1093\/bib\/bbv021","relation":{"has-review":[{"id-type":"doi","id":"10.3410\/f.725441254.793531073","asserted-by":"object"}]},"ISSN":["1467-5463","1477-4054"],"issn-type":[{"value":"1467-5463","type":"print"},{"value":"1477-4054","type":"electronic"}],"subject":[],"published":{"date-parts":[[2015,4,17]]}}}