{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,19]],"date-time":"2026-10-19T15:42:34Z","timestamp":1792424554411,"version":"4.5.0"},"reference-count":42,"publisher":"Elsevier BV","issue":"12","license":[{"start":{"date-parts":[[2020,12,1]],"date-time":"2020-12-01T00:00:00Z","timestamp":1606780800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2020,12,1]],"date-time":"2020-12-01T00:00:00Z","timestamp":1606780800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2021,12,1]],"date-time":"2021-12-01T00:00:00Z","timestamp":1638316800000},"content-version":"vor","delay-in-days":365,"URL":"http:\/\/www.elsevier.com\/open-access\/userlicense\/1.0\/"}],"funder":[{"DOI":"10.13039\/100006093","name":"Patient-Centered Outcomes Research Institute","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100006093","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["clinicalkey.com","clinicalkey.com.au","clinicalkey.es","clinicalkey.fr","clinicalkey.jp","valueinhealthjournal.com","elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Value in Health"],"published-print":{"date-parts":[[2020,12]]},"DOI":"10.1016\/j.jval.2020.08.2093","type":"journal-article","created":{"date-parts":[[2020,10,5]],"date-time":"2020-10-05T18:10:57Z","timestamp":1601921457000},"page":"1630-1638","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":17,"title":["Nature and Predictors of Response Changes in Modified-Delphi Panels"],"prefix":"10.1016","volume":"23","author":[{"given":"Dmitry","family":"Khodyakov","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Christine","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.jval.2020.08.2093_bib1","series-title":"An experimental application of the Delphi method to the use of experts","author":"Dalkey","year":"1962"},{"key":"10.1016\/j.jval.2020.08.2093_bib2","series-title":"The Delphi method: an experimental study of group opinion","author":"Dalkey","year":"1969"},{"key":"10.1016\/j.jval.2020.08.2093_bib3","doi-asserted-by":"crossref","first-page":"373","DOI":"10.1080\/0144341970170401","article-title":"Delphi: a technique to harness expert opinion for critical decision-making tasks in education","volume":"17","author":"Clayton","year":"1997","journal-title":"Educ Psychol"},{"key":"10.1016\/j.jval.2020.08.2093_bib4","doi-asserted-by":"crossref","first-page":"52","DOI":"10.1186\/1471-2288-7-52","article-title":"An exploration of the use of simple statistics to measure consensus and stability in Delphi studies","volume":"7","author":"Holey","year":"2007","journal-title":"BMC Med Res Methodol"},{"key":"10.1016\/j.jval.2020.08.2093_bib5","series-title":"The RAND\/UCLA Appropriateness Method User\u2019s Manual","author":"Fitch","year":"2001"},{"key":"10.1016\/j.jval.2020.08.2093_bib6","doi-asserted-by":"crossref","first-page":"247","DOI":"10.1093\/intqhc\/10.3.247","article-title":"Eliciting expert opinion using the Delphi technique: identifying performance indicators for cardiovascular disease","volume":"10","author":"Normand","year":"1998","journal-title":"Int J Qual Health Care"},{"key":"10.1016\/j.jval.2020.08.2093_bib7","doi-asserted-by":"crossref","first-page":"56","DOI":"10.1186\/s12874-016-0165-8","article-title":"Using the modified Delphi method to establish clinical consensus for the diagnosis and treatment of patients with rotator cuff pathology","volume":"16","author":"Eubank","year":"2016","journal-title":"BMC Med Res Methodol"},{"key":"10.1016\/j.jval.2020.08.2093_bib8","doi-asserted-by":"crossref","DOI":"10.1136\/bmjopen-2016-011780","article-title":"Moving forward through consensus: protocol for a modified Delphi approach to determine the top research priorities in the field of orthopaedic oncology","volume":"6","author":"Schneider","year":"2016","journal-title":"BMJ Open"},{"key":"10.1016\/j.jval.2020.08.2093_bib9","doi-asserted-by":"crossref","first-page":"280","DOI":"10.1016\/0030-5073(73)90052-4","article-title":"A\u00a0comparative study of differences in subjective likelihood estimates made by individuals, interacting groups, Delphi groups, and nominal groups","volume":"9","author":"Gustafson","year":"1973","journal-title":"Organ Behav Hum Perform"},{"key":"10.1016\/j.jval.2020.08.2093_bib10","doi-asserted-by":"crossref","first-page":"218","DOI":"10.1258\/135581906778476553","article-title":"A\u00a0comparison of formal consensus methods used for developing clinical guidelines","volume":"11","author":"Hutchings","year":"2006","journal-title":"J\u00a0Health Serv Res Policy"},{"key":"10.1016\/j.jval.2020.08.2093_bib11","doi-asserted-by":"crossref","first-page":"1426","DOI":"10.1016\/j.techfore.2011.03.021","article-title":"ExpertLens: a system for eliciting opinions from a large pool of non-collocated experts with diverse knowledge","volume":"78","author":"Dalal","year":"2011","journal-title":"Technol Forecast Soc Change"},{"key":"10.1016\/j.jval.2020.08.2093_bib12","doi-asserted-by":"crossref","DOI":"10.1371\/journal.pone.0201378","article-title":"Recruiting and retaining participants in e-Delphi surveys for core outcome set development: evaluating the COMiT\u2019ID study","volume":"13","author":"Hall","year":"2018","journal-title":"PLoS One"},{"key":"10.1016\/j.jval.2020.08.2093_bib13","doi-asserted-by":"crossref","first-page":"160","DOI":"10.1016\/j.jclinepi.2020.04.029","article-title":"Response changes in Delphi processes: why is it important to provide high-quality feedback to Delphi participants?","volume":"125","author":"Khodyakov","year":"2020","journal-title":"J\u00a0Clin Epidemiol"},{"key":"10.1016\/j.jval.2020.08.2093_bib14","doi-asserted-by":"crossref","first-page":"136","DOI":"10.1016\/j.jclinepi.2018.10.011","article-title":"The Delphi method; more research please","volume":"106","author":"Humphrey-Murto","year":"2019","journal-title":"J\u00a0Clin Epidemiol"},{"issue":"1","key":"10.1016\/j.jval.2020.08.2093_bib15","doi-asserted-by":"crossref","first-page":"28","DOI":"10.1186\/s12874-020-0912-8","article-title":"Comparison of different rating scales for the use in Delphi studies: different scales lead to different consensus and show different test-retest reliability","volume":"20","author":"Lange","year":"2020","journal-title":"BMC Med Res Methodol"},{"key":"10.1016\/j.jval.2020.08.2093_bib16","first-page":"470","article-title":"The RAND\/PPMD patient-centeredness method: a novel online approach to engaging patients and their representatives in guideline development","volume":"7","author":"Khodyakov","year":"2019","journal-title":"Eur J Person Cent Healthc"},{"issue":"8","key":"10.1016\/j.jval.2020.08.2093_bib17","doi-asserted-by":"crossref","first-page":"1019","DOI":"10.1177\/0272989X19883631","article-title":"Using an online modified-Delphi approach to engage patients and caregivers in determining the patient-centeredness of Duchenne muscular dystrophy care considerations","volume":"39","author":"Khodyakov","year":"2019","journal-title":"Med Decis Making"},{"key":"10.1016\/j.jval.2020.08.2093_bib18","doi-asserted-by":"crossref","first-page":"354","DOI":"10.1111\/jep.12623","article-title":"Acceptability of an online modified Delphi panel approach for developing health services performance measures: results from 3 panels on arthritis research","volume":"23","author":"Khodyakov","year":"2017","journal-title":"J\u00a0Eval Clin Pract"},{"key":"10.1016\/j.jval.2020.08.2093_bib19","doi-asserted-by":"crossref","first-page":"174","DOI":"10.1186\/1471-2288-11-174","article-title":"Conducting online expert panels: a feasibility and experimental replicability study","volume":"11","author":"Khodyakov","year":"2011","journal-title":"BMC Med Res Methodol"},{"key":"10.1016\/j.jval.2020.08.2093_bib20","doi-asserted-by":"crossref","DOI":"10.1371\/journal.pone.0020476","article-title":"Using and reporting the Delphi method for selecting healthcare quality indicators: a systematic review","volume":"6","author":"Boulkedid","year":"2011","journal-title":"PLoS One"},{"issue":"3","key":"10.1016\/j.jval.2020.08.2093_bib21","doi-asserted-by":"crossref","first-page":"537","DOI":"10.1093\/jamia\/ocw157","article-title":"Comparative analysis of stakeholder experiences with an online approach to prioritizing patient-centered research topics","volume":"24","author":"Khodyakov","year":"2017","journal-title":"J\u00a0Am Med Inform Assoc"},{"key":"10.1016\/j.jval.2020.08.2093_bib22","first-page":"85","article-title":"A\u00a0multivariate technique for multiply imputing missing values using a sequence of regression models","volume":"27","author":"Raghunathan","year":"2001","journal-title":"Survey Methodol"},{"key":"10.1016\/j.jval.2020.08.2093_bib23","doi-asserted-by":"crossref","first-page":"219","DOI":"10.1177\/0962280206074463","article-title":"Multiple imputation of discrete and continuous data by fully conditional specification","volume":"16","author":"Van Buuren","year":"2007","journal-title":"Stat Methods Med Res"},{"key":"10.1016\/j.jval.2020.08.2093_bib24","series-title":"Multiple Imputation for Nonresponse in Surveys","author":"Rubin","year":"2004"},{"key":"10.1016\/j.jval.2020.08.2093_bib25","doi-asserted-by":"crossref","first-page":"171","DOI":"10.1023\/A:1010920819831","article-title":"A\u00a0simple generalisation of the area under the ROC curve for multiple class classification problems","volume":"45","author":"Hand","year":"2001","journal-title":"Mach Learn"},{"key":"10.1016\/j.jval.2020.08.2093_bib26","first-page":"1","article-title":"mice: multivariate imputation by chained equations in R","author":"Buuren","year":"2010","journal-title":"J\u00a0Stat Softw"},{"key":"10.1016\/j.jval.2020.08.2093_bib27","first-page":"1","article-title":"Amelia II: a program for missing data","volume":"45","author":"Honaker","year":"2011","journal-title":"J\u00a0Stat Softw"},{"key":"10.1016\/j.jval.2020.08.2093_bib28","article-title":"Fitting linear mixed-effects models using lme4","volume":"67","author":"Bates","year":"2015","journal-title":"J\u00a0Stat Softw"},{"key":"10.1016\/j.jval.2020.08.2093_bib29","series-title":"mclogit: mixed conditional logit models","author":"Elff","year":"2019"},{"key":"10.1016\/j.jval.2020.08.2093_bib30","doi-asserted-by":"crossref","first-page":"77","DOI":"10.1186\/1471-2105-12-77","article-title":"pROC: an open-source package for R and S+ to analyze and compare ROC curves","volume":"12","author":"Robin","year":"2011","journal-title":"BMC Bioinformatics"},{"key":"10.1016\/j.jval.2020.08.2093_bib31","doi-asserted-by":"crossref","first-page":"131","DOI":"10.1016\/0040-1625(91)90002-W","article-title":"An evaluation of Delphi","volume":"40","author":"Woudenberg","year":"1991","journal-title":"Technol Forecast Soc Change"},{"issue":"4","key":"10.1016\/j.jval.2020.08.2093_bib32","doi-asserted-by":"crossref","first-page":"868","DOI":"10.1111\/hex.12383","article-title":"Collaborative learning framework for online stakeholder engagement","volume":"19","author":"Khodyakov","year":"2016","journal-title":"Health Expect"},{"key":"10.1016\/j.jval.2020.08.2093_bib33","doi-asserted-by":"crossref","first-page":"235","DOI":"10.1016\/0040-1625(91)90039-I","article-title":"Delphi: a reevaluation of research and theory","volume":"39","author":"Rowe","year":"1991","journal-title":"Technol Forecast Soc Change"},{"key":"10.1016\/j.jval.2020.08.2093_bib34","series-title":"The Delphi Method: Techniques and Applications","article-title":"Toward a theory of group estimation","author":"Dalkey","year":"1975"},{"issue":"4157","key":"10.1016\/j.jval.2020.08.2093_bib35","doi-asserted-by":"crossref","first-page":"1124","DOI":"10.1126\/science.185.4157.1124","article-title":"Judgment under uncertainty: heuristics and biases","volume":"185","author":"Tversky","year":"1974","journal-title":"Science"},{"key":"10.1016\/j.jval.2020.08.2093_bib36","doi-asserted-by":"crossref","first-page":"11","DOI":"10.1007\/s40271-019-00389-4","article-title":"Practical considerations in using online modified-Delphi approaches to engage patients and other stakeholders in clinical practice guideline development","volume":"13","author":"Khodyakov","year":"2020","journal-title":"Patient"},{"key":"10.1016\/j.jval.2020.08.2093_bib37","doi-asserted-by":"crossref","first-page":"67","DOI":"10.1016\/0040-1625(80)90074-8","article-title":"Individual stability in Delphi studies","volume":"16","author":"Chaffin","year":"1980","journal-title":"Technol Forecast Soc Change"},{"key":"10.1016\/j.jval.2020.08.2093_bib38","doi-asserted-by":"crossref","first-page":"83","DOI":"10.1016\/0040-1625(79)90007-6","article-title":"Stability and agreement criteria for the termination of Delphi studies","volume":"13","author":"Dajani","year":"1979","journal-title":"Technol Forecast Soc Change"},{"key":"10.1016\/j.jval.2020.08.2093_bib39","doi-asserted-by":"crossref","first-page":"195","DOI":"10.1016\/S0020-7489(00)00044-4","article-title":"A\u00a0critical review of the Delphi technique as a research methodology for nursing","volume":"38","author":"Keeney","year":"2001","journal-title":"Int J Nurs Stud"},{"key":"10.1016\/j.jval.2020.08.2093_bib40","doi-asserted-by":"crossref","first-page":"79","DOI":"10.1016\/0040-1625(87)90024-2","article-title":"Methods for analyzing data from Delphi panels: some evidence from a forecasting study","volume":"31","author":"Dietz","year":"1987","journal-title":"Technol Forecast Soc Change"},{"key":"10.1016\/j.jval.2020.08.2093_bib41","doi-asserted-by":"crossref","first-page":"89","DOI":"10.1016\/0040-1625(83)90073-2","article-title":"The Delphi technique: an experimental evaluation","volume":"23","author":"Riggs","year":"1983","journal-title":"Technol Forecast Soc Change"},{"key":"10.1016\/j.jval.2020.08.2093_bib42","doi-asserted-by":"crossref","first-page":"173","DOI":"10.1002\/for.3980030205","article-title":"An examination of factors contributing to Delphi accuracy","volume":"3","author":"Parent\u00e9","year":"1984","journal-title":"J\u00a0Forecast"}],"container-title":["Value in Health"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1098301520343539?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1098301520343539?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,10,22]],"date-time":"2025-10-22T01:21:37Z","timestamp":1761096097000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S1098301520343539"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,12]]},"references-count":42,"journal-issue":{"issue":"12","published-print":{"date-parts":[[2020,12]]}},"alternative-id":["S1098301520343539"],"URL":"https:\/\/doi.org\/10.1016\/j.jval.2020.08.2093","relation":{},"ISSN":["1098-3015"],"issn-type":[{"value":"1098-3015","type":"print"}],"subject":[],"published":{"date-parts":[[2020,12]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Nature and Predictors of Response Changes in Modified-Delphi Panels","name":"articletitle","label":"Article Title"},{"value":"Value in Health","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.jval.2020.08.2093","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2020 ISPOR-The Professional Society for Health Economics and Outcomes Research. Published by Elsevier Inc.","name":"copyright","label":"Copyright"}]}}