{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,7]],"date-time":"2026-03-07T19:11:35Z","timestamp":1772910695323,"version":"3.50.1"},"reference-count":43,"publisher":"Oxford University Press (OUP)","issue":"7","license":[{"start":{"date-parts":[[2022,4,15]],"date-time":"2022-04-15T00:00:00Z","timestamp":1649980800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000092","name":"National Library of Medicine","doi-asserted-by":"publisher","award":["5R01LM009886-11"],"award-info":[{"award-number":["5R01LM009886-11"]}],"id":[{"id":"10.13039\/100000092","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100006108","name":"National Center for Advancing Translational Sciences","doi-asserted-by":"publisher","award":["UL1TR001873"],"award-info":[{"award-number":["UL1TR001873"]}],"id":[{"id":"10.13039\/100006108","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100006108","name":"National Center for Advancing Translational Sciences","doi-asserted-by":"publisher","award":["1OT2TR003434-01"],"award-info":[{"award-number":["1OT2TR003434-01"]}],"id":[{"id":"10.13039\/100006108","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000056","name":"National Institute of Nursing Research","doi-asserted-by":"publisher","award":["T32NR007969"],"award-info":[{"award-number":["T32NR007969"]}],"id":[{"id":"10.13039\/100000056","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000056","name":"National Institute of Nursing Research","doi-asserted-by":"publisher","award":["K24NR018621"],"award-info":[{"award-number":["K24NR018621"]}],"id":[{"id":"10.13039\/100000056","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000133","name":"Agency for Healthcare Research and Quality","doi-asserted-by":"publisher","award":["1R36HS028752"],"award-info":[{"award-number":["1R36HS028752"]}],"id":[{"id":"10.13039\/100000133","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022,6,14]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:sec>\n                  <jats:title>Objective<\/jats:title>\n                  <jats:p>To combine machine efficiency and human intelligence for converting complex clinical trial eligibility criteria text into cohort queries.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Materials and Methods<\/jats:title>\n                  <jats:p>Criteria2Query (C2Q) 2.0 was developed to enable real-time user intervention for criteria selection and simplification, parsing error correction, and concept mapping. The accuracy, precision, recall, and F1 score of enhanced modules for negation scope detection, temporal and value normalization were evaluated using a previously curated gold standard, the annotated eligibility criteria of 1010 COVID-19 clinical trials. The usability and usefulness were evaluated by 10 research coordinators in a task-oriented usability evaluation using 5 Alzheimer\u2019s disease trials. Data were collected by user interaction logging, a demographic questionnaire, the Health Information Technology Usability Evaluation Scale (Health-ITUES), and a feature-specific questionnaire.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Results<\/jats:title>\n                  <jats:p>The accuracies of negation scope detection, temporal and value normalization were 0.924, 0.916, and 0.966, respectively. C2Q 2.0 achieved a moderate usability score (3.84 out of 5) and a high learnability score (4.54 out of 5). On average, 9.9 modifications were made for a clinical study. Experienced researchers made more modifications than novice researchers. The most frequent modification was deletion (5.35 per study). Furthermore, the evaluators favored cohort queries resulting from modifications (score 4.1 out of 5) and the user engagement features (score 4.3 out of 5).<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Discussion and Conclusion<\/jats:title>\n                  <jats:p>Features to engage domain experts and to overcome the limitations in automated machine output are shown to be useful and user-friendly. We concluded that human\u2013computer collaboration is key to improving the adoption and user-friendliness of natural language processing.<\/jats:p>\n               <\/jats:sec>","DOI":"10.1093\/jamia\/ocac051","type":"journal-article","created":{"date-parts":[[2022,3,29]],"date-time":"2022-03-29T19:14:20Z","timestamp":1648581260000},"page":"1161-1171","source":"Crossref","is-referenced-by-count":24,"title":["Combining human and machine intelligence for clinical trial eligibility querying"],"prefix":"10.1093","volume":"29","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2681-1931","authenticated-orcid":false,"given":"Yilu","family":"Fang","sequence":"first","affiliation":[{"name":"Department of Biomedical Informatics, Columbia University , New York, New York, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4318-5987","authenticated-orcid":false,"given":"Betina","family":"Idnay","sequence":"additional","affiliation":[{"name":"School of Nursing, Columbia University , New York, New York, USA"},{"name":"Department of Neurology, Columbia University , New York, New York, USA"}]},{"given":"Yingcheng","family":"Sun","sequence":"additional","affiliation":[{"name":"Department of Biomedical Informatics, Columbia University , New York, New York, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1975-1272","authenticated-orcid":false,"given":"Hao","family":"Liu","sequence":"additional","affiliation":[{"name":"Department of Biomedical Informatics, Columbia University , New York, New York, USA"}]},{"given":"Zhehuan","family":"Chen","sequence":"additional","affiliation":[{"name":"Department of Biomedical Informatics, Columbia University , New York, New York, USA"}]},{"given":"Karen","family":"Marder","sequence":"additional","affiliation":[{"name":"Department of Neurology, Columbia University , New York, New York, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5274-4672","authenticated-orcid":false,"given":"Hua","family":"Xu","sequence":"additional","affiliation":[{"name":"School of Biomedical Informatics, The University of Texas Health Science Center at Houston , Houston, Texas, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2184-4045","authenticated-orcid":false,"given":"Rebecca","family":"Schnall","sequence":"additional","affiliation":[{"name":"School of Nursing, Columbia University , New York, New York, USA"},{"name":"Heilbrunn Department of Population and Family Health, Mailman School of Public Health, Columbia University , New York, New York, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9624-0214","authenticated-orcid":false,"given":"Chunhua","family":"Weng","sequence":"additional","affiliation":[{"name":"Department of Biomedical Informatics, Columbia University , New York, New York, USA"}]}],"member":"286","published-online":{"date-parts":[[2022,4,15]]},"reference":[{"key":"2022061415515197400_ocac051-B1","doi-asserted-by":"crossref","first-page":"8","DOI":"10.1186\/1478-4505-13-8","article-title":"Barriers and opportunities for enhancing patient recruitment and retention in clinical research: findings from an interview study in an NHS academic health science centre","volume":"13","author":"Adams","year":"2015","journal-title":"Health Res Policy Syst"},{"key":"2022061415515197400_ocac051-B2","doi-asserted-by":"crossref","first-page":"105","DOI":"10.1200\/EDBK_243729","article-title":"Overcoming barriers to clinical trial enrollment","volume":"39","author":"Nipp","year":"2019","journal-title":"Am Soc Clin Oncol Educ Book"},{"issue":"3","key":"2022061415515197400_ocac051-B3","doi-asserted-by":"crossref","first-page":"316","DOI":"10.1197\/jamia.M3005","article-title":"Clinical research informatics: challenges, opportunities and definition for an emerging domain","volume":"16","author":"Embi","year":"2009","journal-title":"J Am Med Inform Assoc"},{"key":"2022061415515197400_ocac051-B4","doi-asserted-by":"crossref","first-page":"S211","DOI":"10.1016\/j.jbi.2015.09.008","article-title":"Textual inference for eligibility criteria resolution in clinical trials","volume":"58 (Suppl","author":"Shivade","year":"2015","journal-title":"J Biomed Inform"},{"key":"2022061415515197400_ocac051-B5","first-page":"564","article-title":"Automatic selection of clinical trials based on a semantic web approach","volume":"216","author":"Cuggia","year":"2015","journal-title":"Stud Health Technol Inform"},{"issue":"1","key":"2022061415515197400_ocac051-B6","doi-asserted-by":"crossref","first-page":"151","DOI":"10.1177\/0962280218823036","article-title":"Reduction of recruitment costs in preclinical AD trials: validation of automatic pre-screening algorithm for brain amyloidosis","volume":"29","author":"Ansart","year":"2020","journal-title":"Stat Methods Med Res"},{"issue":"11","key":"2022061415515197400_ocac051-B7","doi-asserted-by":"crossref","first-page":"706","DOI":"10.1016\/j.tips.2015.08.007","article-title":"Optimizing clinical research participant selection with informatics","volume":"36","author":"Weng","year":"2015","journal-title":"Trends Pharmacol Sci"},{"issue":"11","key":"2022061415515197400_ocac051-B8","doi-asserted-by":"crossref","first-page":"1172","DOI":"10.1093\/jamia\/ocz079","article-title":"Hybrid bag of approaches to characterize selection criteria for cohort identification","volume":"26","author":"Vydiswaran","year":"2019","journal-title":"J Am Med Inform Assoc"},{"issue":"1","key":"2022061415515197400_ocac051-B9","doi-asserted-by":"crossref","first-page":"166","DOI":"10.1136\/amiajnl-2014-002887","article-title":"Automated clinical trial eligibility prescreening: increasing the efficiency of patient identification for clinical trials in the emergency department","volume":"22","author":"Ni","year":"2015","journal-title":"J Am Med Inform Assoc"},{"key":"2022061415515197400_ocac051-B10","doi-asserted-by":"crossref","first-page":"28","DOI":"10.1186\/s12911-015-0149-3","article-title":"Increasing the efficiency of trial-patient matching: automated clinical trial eligibility pre-screening for pediatric oncology patients","volume":"15","author":"Ni","year":"2015","journal-title":"BMC Med Inform Decis Mak"},{"issue":"3","key":"2022061415515197400_ocac051-B11","doi-asserted-by":"crossref","first-page":"266","DOI":"10.3414\/ME15-01-0112","article-title":"Valx: a system for extracting and structuring numeric lab test comparison statements from text","volume":"55","author":"Hao","year":"2016","journal-title":"Methods Inf Med"},{"issue":"6","key":"2022061415515197400_ocac051-B12","doi-asserted-by":"crossref","first-page":"1062","DOI":"10.1093\/jamia\/ocx019","article-title":"EliIE: an open-source information extraction system for clinical trial eligibility criteria","volume":"24","author":"Kang","year":"2017","journal-title":"J Am Med Inform Assoc"},{"issue":"4","key":"2022061415515197400_ocac051-B13","doi-asserted-by":"crossref","first-page":"294","DOI":"10.1093\/jamia\/ocy178","article-title":"Criteria2Query: a natural language interface to clinical databases for cohort definition","volume":"26","author":"Yuan","year":"2019","journal-title":"J Am Med Inform Assoc"},{"key":"2022061415515197400_ocac051-B14","author":"Tseo","year":"2020"},{"key":"2022061415515197400_ocac051-B15","author":"Liu","year":"2020"},{"key":"2022061415515197400_ocac051-B16","author":"Tian","year":"2021"},{"key":"2022061415515197400_ocac051-B17","author":"Liu","year":"2021"},{"key":"2022061415515197400_ocac051-B18","author":"Chen","year":"2020"},{"key":"2022061415515197400_ocac051-B19","first-page":"269","article-title":"BERT-based ranking for biomedical entity normalization","volume":"2020","author":"Ji","year":"2020","journal-title":"AMIA Summits Transl Sci Proc"},{"issue":"21","key":"2022061415515197400_ocac051-B20","doi-asserted-by":"crossref","first-page":"3856","DOI":"10.1093\/bioinformatics\/btab474","article-title":"Medical concept normalization in clinical trials with drug and disease representation learning","volume":"37","author":"Miftahutdinov","year":"2021","journal-title":"Bioinformatics"},{"key":"2022061415515197400_ocac051-B21","first-page":"46","article-title":"Analysis of eligibility criteria complexity in clinical trials","volume":"2010","author":"Ross","year":"2010","journal-title":"Summit Transl Bioinform"},{"key":"2022061415515197400_ocac051-B22","first-page":"394","author":"Li"},{"issue":"1","key":"2022061415515197400_ocac051-B23","doi-asserted-by":"crossref","first-page":"197","DOI":"10.1093\/jamia\/ocab228","article-title":"A systematic review on natural language processing systems for eligibility prescreening in clinical research","volume":"29","author":"Idnay","year":"2021","journal-title":"J Am Med Inform Assoc"},{"key":"2022061415515197400_ocac051-B24","first-page":"984","article-title":"Participatory design of a clinical trial eligibility criteria simplification method","volume":"281","author":"Fang","year":"2021","journal-title":"Stud Health Technol Inform"},{"key":"2022061415515197400_ocac051-B25","author":"Khandelwal","year":"2020:"},{"issue":"1","key":"2022061415515197400_ocac051-B26","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3458754","article-title":"Domain-specific language model pretraining for biomedical natural language processing","volume":"3","author":"Gu","year":"2021","journal-title":"ACM Trans Comput Healthc"},{"key":"2022061415515197400_ocac051-B27","author":"Britto","year":"2020"},{"issue":"S11","key":"2022061415515197400_ocac051-B28","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/1471-2105-9-S11-S9","article-title":"The BioScope corpus: biomedical texts annotated for uncertainty, negation and their scopes","volume":"9","author":"Vincze","year":"2008","journal-title":"BMC Bioinform"},{"key":"2022061415515197400_ocac051-B29","first-page":"3735","author":"Chang"},{"key":"2022061415515197400_ocac051-B30","author":"Manning","year":"2014"},{"key":"2022061415515197400_ocac051-B31","doi-asserted-by":"crossref","first-page":"103790","DOI":"10.1016\/j.jbi.2021.103790","article-title":"Building an OMOP common data model-compliant annotated corpus for COVID-19 clinical trials","volume":"118","author":"Sun","year":"2021","journal-title":"J Biomed Inform"},{"issue":"3","key":"2022061415515197400_ocac051-B32","doi-asserted-by":"crossref","first-page":"e33677","DOI":"10.1371\/journal.pone.0033677","article-title":"The database for aggregate analysis of ClinicalTrials. gov (AACT) and subsequent regrouping by clinical specialty","volume":"7","author":"Tasneem","year":"2012","journal-title":"PLoS One"},{"issue":"2","key":"2022061415515197400_ocac051-B33","doi-asserted-by":"crossref","first-page":"141","DOI":"10.1177\/004912418101000205","article-title":"Snowball sampling: problems and techniques of chain referral sampling","volume":"10","author":"Biernacki","year":"1981","journal-title":"Sociol Methods Res"},{"issue":"5","key":"2022061415515197400_ocac051-B34","doi-asserted-by":"crossref","first-page":"130","DOI":"10.1145\/1735223.1735255","article-title":"Number of people required for usability evaluation","volume":"53","author":"Hwang","year":"2010","journal-title":"Commun ACM"},{"key":"2022061415515197400_ocac051-B35","first-page":"917","article-title":"Development of a customizable Health IT Usability Evaluation Scale","volume":"2010","author":"Yen","year":"2010","journal-title":"AMIA Annu Symp Proc"},{"issue":"e2","key":"2022061415515197400_ocac051-B36","doi-asserted-by":"crossref","first-page":"e241\u20138","DOI":"10.1136\/amiajnl-2013-001811","article-title":"Examining construct and predictive validity of the Health-IT Usability Evaluation Scale: confirmatory factor analysis and structural equation modeling results","volume":"21","author":"Yen","year":"2014","journal-title":"J Am Med Inform Assoc"},{"issue":"1","key":"2022061415515197400_ocac051-B37","doi-asserted-by":"crossref","first-page":"e4","DOI":"10.2196\/mhealth.8851","article-title":"Health Information Technology Usability Evaluation Scale (Health-ITUES) for usability assessment of mobile health technology: validation study","volume":"6","author":"Schnall","year":"2018","journal-title":"JMIR Mhealth Uhealth"},{"key":"2022061415515197400_ocac051-B38","doi-asserted-by":"crossref","first-page":"104713","DOI":"10.1016\/j.ijmedinf.2022.104713","article-title":"Establishing a usability cut-point for the Health Information Technology Usability Evaluation Scale (Health-ITUES)","volume":"160","author":"Loh","year":"2022","journal-title":"Int J Med Inform"},{"issue":"2","key":"2022061415515197400_ocac051-B39","first-page":"3","article-title":"Measuring usability with the use questionnaire","volume":"8","author":"Lund","year":"2001","journal-title":"Usability Interface"},{"key":"2022061415515197400_ocac051-B40","doi-asserted-by":"crossref","first-page":"7862672","DOI":"10.1155\/2017\/7862672","article-title":"Computer-aided clinical trial recruitment based on domain-specific language translation: a case study of retinopathy of prematurity","volume":"2017","author":"Zhang","year":"2017","journal-title":"J Healthc Eng"},{"issue":"11","key":"2022061415515197400_ocac051-B41","doi-asserted-by":"crossref","first-page":"1163","DOI":"10.1093\/jamia\/ocz163","article-title":"Cohort selection for clinical trials: n2c2 2018 shared task track 1","volume":"26","author":"Stubbs","year":"2019","journal-title":"J Am Med Inform Assoc"},{"key":"2022061415515197400_ocac051-B42","doi-asserted-by":"crossref","first-page":"S189","DOI":"10.1016\/j.jbi.2015.07.008","article-title":"Ease of adoption of clinical natural language processing software: an evaluation of five systems","volume":"58 (Suppl","author":"Zheng","year":"2015","journal-title":"J Biomed Inform"},{"key":"2022061415515197400_ocac051-B43","article-title":"Evaluating Natural Language Processing Systems: An Analysis and Review. Berlin, Heidelberg: Springer-Verlag; 1995","author":"Jones"}],"container-title":["Journal of the American Medical Informatics Association"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/29\/7\/1161\/44062110\/ocac051.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/29\/7\/1161\/44062110\/ocac051.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,6,14]],"date-time":"2022-06-14T15:52:32Z","timestamp":1655221952000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamia\/article\/29\/7\/1161\/6569054"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,4,15]]},"references-count":43,"journal-issue":{"issue":"7","published-online":{"date-parts":[[2022,4,15]]},"published-print":{"date-parts":[[2022,6,14]]}},"URL":"https:\/\/doi.org\/10.1093\/jamia\/ocac051","relation":{},"ISSN":["1527-974X"],"issn-type":[{"value":"1527-974X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2022,7,1]]},"published":{"date-parts":[[2022,4,15]]}}}