{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,22]],"date-time":"2026-01-22T13:09:33Z","timestamp":1769087373879,"version":"3.49.0"},"reference-count":18,"publisher":"Oxford University Press (OUP)","issue":"11","license":[{"start":{"date-parts":[[2019,7,30]],"date-time":"2019-07-30T00:00:00Z","timestamp":1564444800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/academic.oup.com\/journals\/pages\/open_access\/funder_policies\/chorus\/standard_publication_model"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2019,11,1]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:sec>\n                  <jats:title>Objective<\/jats:title>\n                  <jats:p>We assessed whether machine learning can be utilized to allow efficient extraction of infectious disease activity information from online media reports.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Materials and Methods<\/jats:title>\n                  <jats:p>We curated a data set of labeled media reports (n\u2009=\u20098322) indicating which articles contain updates about disease activity. We trained a classifier on this data set. To validate our system, we used a held out test set and compared our articles to the World Health Organization Disease Outbreak News reports.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Results<\/jats:title>\n                  <jats:p>Our classifier achieved a recall and precision of 88.8% and 86.1%, respectively. The overall surveillance system detected 94% of the outbreaks identified by the WHO covered by online media (89%) and did so 43.4 (IQR: 9.5\u201361) days earlier on average.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Discussion<\/jats:title>\n                  <jats:p>We constructed a global real-time disease activity database surveilling 114 illnesses and syndromes. We must further assess our system for bias, representativeness, granularity, and accuracy.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Conclusion<\/jats:title>\n                  <jats:p>Machine learning, natural language processing, and human expertise can be used to efficiently identify disease activity from digital media reports.<\/jats:p>\n               <\/jats:sec>","DOI":"10.1093\/jamia\/ocz112","type":"journal-article","created":{"date-parts":[[2019,6,4]],"date-time":"2019-06-04T19:11:55Z","timestamp":1559675515000},"page":"1355-1359","source":"Crossref","is-referenced-by-count":21,"title":["Development of a global infectious disease activity database using natural language processing, machine learning, and human expertise"],"prefix":"10.1093","volume":"26","author":[{"given":"Joshua","family":"Feldman","sequence":"first","affiliation":[{"name":"Harvard University, School of Engineering and Applied Sciences, Cambridge, Massachusetts, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0831-842X","authenticated-orcid":false,"given":"Andrea","family":"Thomas-Bachli","sequence":"additional","affiliation":[{"name":"Li Ka Shing Knowledge Institute, St. Michaels Hospital, Toronto, Ontario, Canada"},{"name":"BlueDot, Toronto, Ontario, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jack","family":"Forsyth","sequence":"additional","affiliation":[{"name":"Li Ka Shing Knowledge Institute, St. Michaels Hospital, Toronto, Ontario, Canada"},{"name":"BlueDot, Toronto, Ontario, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zaki Hasnain","family":"Patel","sequence":"additional","affiliation":[{"name":"Li Ka Shing Knowledge Institute, St. Michaels Hospital, Toronto, Ontario, Canada"},{"name":"BlueDot, Toronto, Ontario, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kamran","family":"Khan","sequence":"additional","affiliation":[{"name":"Li Ka Shing Knowledge Institute, St. Michaels Hospital, Toronto, Ontario, Canada"},{"name":"BlueDot, Toronto, Ontario, Canada"},{"name":"Division of Infectious Diseases, Department of Medicine, University of Toronto, Toronto, Ontario, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"286","published-online":{"date-parts":[[2019,7,30]]},"reference":[{"key":"2020110613072016000_ocz112-B1","doi-asserted-by":"crossref","first-page":"77","DOI":"10.1016\/j.ijid.2017.07.020","article-title":"Utility and potential of rapid epidemic intelligence from internet-based sources","volume":"63","author":"Yan","year":"2017","journal-title":"Int J Infect Dis"},{"key":"2020110613072016000_ocz112-B2","doi-asserted-by":"crossref","first-page":"15","DOI":"10.1016\/j.ijmedinf.2017.01.019","article-title":"Digital disease detection: a systematic review of event-based internet biosurveillance systems","volume":"101","author":"O\u2019Shea","year":"2017","journal-title":"Int J Med Inform"},{"issue":"3","key":"2020110613072016000_ocz112-B3","doi-asserted-by":"crossref","first-page":"e57252","DOI":"10.1371\/journal.pone.0057252","article-title":"Evaluation of epidemic intelligence systems integrated in the early alerting and reporting project for the detection of A\/H5N1 influenza events","volume":"8","author":"Barboza","year":"2013","journal-title":"PLoS ONE"},{"issue":"3","key":"2020110613072016000_ocz112-B4","doi-asserted-by":"crossref","first-page":"223","DOI":"10.1111\/j.1865-1682.2011.01258.x","article-title":"Comparison of web-based biosecurity intelligence systems: BioCaster, EpiSPIDER and HealthMap","volume":"59","author":"Lyon","year":"2012","journal-title":"Transbound Emerg Dis"},{"issue":"7","key":"2020110613072016000_ocz112-B5","doi-asserted-by":"crossref","first-page":"1184","DOI":"10.3201\/eid1807.120249","article-title":"Timeliness of nongovernmental versus governmental global outbreak communications","volume":"18","author":"Mondor","year":"2012","journal-title":"Emerg Infect Dis"},{"issue":"7","key":"2020110613072016000_ocz112-B6","doi-asserted-by":"crossref","first-page":"1147","DOI":"10.3201\/eid1808.120055","article-title":"Electronic event-based surveillance for monitoring dengue, Latin America","volume":"18","author":"Hoen","year":"2012","journal-title":"Emerg Infect Dis"},{"key":"2020110613072016000_ocz112-B7","doi-asserted-by":"crossref","first-page":"S375","DOI":"10.1093\/infdis\/jiw400","article-title":"Big data for infectious disease surveillance and modeling","volume":"214 (Suppl 4)","author":"Bansal","year":"2016","journal-title":"J Infect Dis"},{"issue":"4","key":"2020110613072016000_ocz112-B8","doi-asserted-by":"crossref","first-page":"e1001413.","DOI":"10.1371\/journal.pmed.1001413","article-title":"Big data opportunities for global infectious disease surveillance","volume":"10","author":"Hay","year":"2013","journal-title":"PLoS Med"},{"key":"2020110613072016000_ocz112-B9","author":"Leetaru","year":"2013"},{"key":"2020110613072016000_ocz112-B10","author":"The GDELT Project 2013\u20132018","year":"2018"},{"key":"2020110613072016000_ocz112-B11","author":"World Health Organization","year":"2018"},{"key":"2020110613072016000_ocz112-B12","author":"Wu","year":"2016"},{"issue":"11","key":"2020110613072016000_ocz112-B13","doi-asserted-by":"crossref","first-page":"1006","DOI":"10.1111\/1469-0691.12273","article-title":"An overview of internet biosurveillance","volume":"19","author":"Hartley","year":"2013","journal-title":"Clin Microbiol Infect"},{"issue":"1","key":"2020110613072016000_ocz112-B14","doi-asserted-by":"crossref","first-page":"21621","DOI":"10.3402\/ehtj.v6i0.21621","article-title":"Quantifying the effect of media limitations on outbreak data in a global online web-crawling epidemic intelligence system, 2008\u20132011","volume":"6","author":"Scales","year":"2013","journal-title":"J Emerg Health Threats"},{"issue":"10","key":"2020110613072016000_ocz112-B15","doi-asserted-by":"crossref","first-page":"e110236","DOI":"10.1371\/journal.pone.0110236","article-title":"Evaluation of local media surveillance for improved disease recognition and monitoring in global hotspot regions","volume":"9","author":"Schwind","year":"2014","journal-title":"PLoS One"},{"key":"2020110613072016000_ocz112-B16","author":"International Society for Infectious Diseases","year":"2010"},{"key":"2020110613072016000_ocz112-B17","doi-asserted-by":"crossref","first-page":"40841","DOI":"10.1038\/srep40841","article-title":"Temporal topic modeling to assess associations between news trends and infectious disease outbreaks","volume":"7","author":"Ghosh","year":"2017","journal-title":"Sci Rep"},{"key":"2020110613072016000_ocz112-B18","author":"BlueDot Inc","year":"2019"}],"container-title":["Journal of the American Medical Informatics Association"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/academic.oup.com\/jamia\/article-pdf\/26\/11\/1355\/34151721\/ocz112.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"http:\/\/academic.oup.com\/jamia\/article-pdf\/26\/11\/1355\/34151721\/ocz112.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,11,6]],"date-time":"2020-11-06T19:13:55Z","timestamp":1604690035000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamia\/article\/26\/11\/1355\/5541002"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,7,30]]},"references-count":18,"journal-issue":{"issue":"11","published-online":{"date-parts":[[2019,7,30]]},"published-print":{"date-parts":[[2019,11,1]]}},"URL":"https:\/\/doi.org\/10.1093\/jamia\/ocz112","relation":{},"ISSN":["1067-5027","1527-974X"],"issn-type":[{"value":"1067-5027","type":"print"},{"value":"1527-974X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2019,11]]},"published":{"date-parts":[[2019,7,30]]}}}