{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,2]],"date-time":"2026-04-02T10:06:55Z","timestamp":1775124415552,"version":"3.50.1"},"reference-count":60,"publisher":"Springer Science and Business Media LLC","issue":"7890","license":[{"start":{"date-parts":[[2021,12,8]],"date-time":"2021-12-08T00:00:00Z","timestamp":1638921600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,12,8]],"date-time":"2021-12-08T00:00:00Z","timestamp":1638921600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nature"],"published-print":{"date-parts":[[2021,12,23]]},"DOI":"10.1038\/s41586-021-04198-4","type":"journal-article","created":{"date-parts":[[2021,12,8]],"date-time":"2021-12-08T11:05:18Z","timestamp":1638961518000},"page":"695-700","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":192,"title":["Unrepresentative big surveys significantly overestimated US vaccine uptake"],"prefix":"10.1038","volume":"600","author":[{"given":"Valerie C.","family":"Bradley","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5687-2647","authenticated-orcid":false,"given":"Shiro","family":"Kuriwaki","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6468-5908","authenticated-orcid":false,"given":"Michael","family":"Isakov","sequence":"additional","affiliation":[]},{"given":"Dino","family":"Sejdinovic","sequence":"additional","affiliation":[]},{"given":"Xiao-Li","family":"Meng","sequence":"additional","affiliation":[]},{"given":"Seth","family":"Flaxman","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2021,12,8]]},"reference":[{"key":"4198_CR1","doi-asserted-by":"publisher","first-page":"685","DOI":"10.1214\/18-AOAS1161SF","volume":"12","author":"X-L Meng","year":"2018","unstructured":"Meng, X.-L. Statistical paradises and paradoxes in big data (I): Law of large populations, big data paradox, and the 2016 US presidential election. Ann. Appl. Stat. 12, 685\u2013726 (2018).","journal-title":"Ann. Appl. Stat."},{"key":"4198_CR2","unstructured":"Barkay, N. et al. Weights and methodology brief for the COVID-19 Symptom Survey by University of Maryland and Carnegie Mellon University, in partnership with Facebook. Preprint at https:\/\/arxiv.org\/abs\/2009.14675 (2020)."},{"key":"4198_CR3","first-page":"159","volume":"14","author":"F Kreuter","year":"2020","unstructured":"Kreuter, F. et al. Partnering with Facebook on a university-based rapid turn-around global survey. Surv. Res. Methods 14, 159\u2013163 (2020).","journal-title":"Surv. Res. Methods"},{"key":"4198_CR4","unstructured":"Fields, J. F. et al. Design and Operation of the 2020 Household Pulse Survey (U.S. Census Bureau, 2020)."},{"key":"4198_CR5","unstructured":"Jackson, C., Newall, M. & Yi, J. Axios Ipsos Coronavirus Index (2021)."},{"key":"4198_CR6","unstructured":"American Association for Public Opinion Research (AAPOR). Best practices for survey research. https:\/\/www.aapor.org\/Standards-Ethics\/Best-Practices.aspx (2021)."},{"key":"4198_CR7","doi-asserted-by":"publisher","first-page":"170","DOI":"10.1509\/jppm.20.2.170.17368","volume":"20","author":"M Hastak","year":"2001","unstructured":"Hastak, M., Mazis, M. B. & Morris, L. A. The role of consumer surveys in public policy decision making. J. Public Policy Mark. 20, 170\u2013185 (2001).","journal-title":"J. Public Policy Mark."},{"key":"4198_CR8","doi-asserted-by":"publisher","first-page":"759","DOI":"10.15585\/mmwr.mm7020e3","volume":"70","author":"BP Murthy","year":"2021","unstructured":"B. P. Murthy, et al. Disparities in COVID-19 vaccination coverage between urban and rural counties: United States, December 14, 2020\u2013April 10, 2021. MMWR Morb. Mortal. Wkly Rep. 70, 759\u2013764 (2021).","journal-title":"MMWR Morb. Mortal. Wkly Rep."},{"key":"4198_CR9","unstructured":"Arrieta, A., Gakidou, E., Larson, H., Mullany, E. & Troeger, C. Through understanding and empathy, we can convince women to get the COVID-19 vaccine. Think Global Health https:\/\/www.thinkglobalhealth.org\/article\/through-understanding-and-empathy-we-can-convince-women-get-covid-19-vaccine (2021)."},{"key":"4198_CR10","unstructured":"Japec, L. et al. AAPOR Report on Big Data (American Association of Public Opinion Researchers, 2015)."},{"key":"4198_CR11","unstructured":"Reinhart, A., Kim, E., Garcia, A. & LaRocca, S. Using the COVID-19 Symptom Survey to track vaccination uptake and sentiment in the United States. CMU Delphi Group https:\/\/delphi.cmu.edu\/blog\/2021\/01\/28\/using-the-covid-19-symptom-survey-to-track-vaccination-uptake-and-sentiment-in-the-united-states (2021)."},{"key":"4198_CR12","unstructured":"Mayer-Sch\u00f6nberger, V. & Cukier, K. Big Data: A Revolution That Will Transform How We Live, Work, and Think (Houghton Mifflin Harcourt, 2013)."},{"key":"4198_CR13","unstructured":"CDC. Trends in number of COVID-19 vaccinations (2021)."},{"key":"4198_CR14","unstructured":"Nguyen, K. H. et al. Comparison of COVID-19 vaccination coverage estimates from the Household Pulse Survey, Omnibus Panel Surveys, and COVID-19 vaccine administration data, United States, March 2021. CDC AdultVaxView https:\/\/www.cdc.gov\/vaccines\/imz-managers\/coverage\/adultvaxview\/pubs-resources\/covid19-coverage-estimates-comparison.html (2021)."},{"key":"4198_CR15","unstructured":"Santibanez, T. A. et al. Sociodemographic factors associated with receipt of COVID-19 vaccination and intent to definitely get vaccinated, adults aged 18 years or above\u2014Household Pulse Survey, United States, April 28\u2013May 10, 2021. CDC AdultVaxView https:\/\/www.cdc.gov\/vaccines\/imz-managers\/coverage\/adultvaxview\/pubs-resources\/sociodemographic-factors-covid19-vaccination.html (2021)."},{"key":"4198_CR16","doi-asserted-by":"publisher","first-page":"13","DOI":"10.2307\/1403202","volume":"47","author":"W Kruskal","year":"1979","unstructured":"Kruskal, W. & Mosteller, F. Representative sampling, I: Non-scientific literature. Int. Stat. Rev. 47,13\u201324 (1979).","journal-title":"Int. Stat. Rev."},{"key":"4198_CR17","doi-asserted-by":"publisher","first-page":"111","DOI":"10.2307\/1402564","volume":"47","author":"W Kruskal","year":"1979","unstructured":"Kruskal, W. & Mosteller, F. Representative sampling, II: Scientific literature, excluding statistics. Int. Stat. Rev. 47, 111\u2013127 (1979).","journal-title":"Int. Stat. Rev."},{"key":"4198_CR18","doi-asserted-by":"publisher","first-page":"245","DOI":"10.2307\/1402647","volume":"47","author":"W Kruskal","year":"1979","unstructured":"Kruskal, W. & Mosteller, F. Representative sampling, III: The current statistical literature. Int. Stat. Rev. 47, 245\u2013265 (1979).","journal-title":"Int. Stat. Rev."},{"key":"4198_CR19","doi-asserted-by":"publisher","first-page":"169","DOI":"10.2307\/1403151","volume":"48","author":"W Kruskal","year":"1980","unstructured":"Kruskal, W. & Mosteller, F. Representative sampling, IV: The history of the concept in statistics, 1895\u20131939. Int. Stat. Rev. 48, 169\u2013195 (1980).","journal-title":"Int. Stat. Rev."},{"key":"4198_CR20","unstructured":"American Association for Public Opinion Research (AAPOR). Margin of sampling error\/credibility interval. https:\/\/www.aapor.org\/Education-Resources\/Election-Polling-Resources\/Margin-of-Sampling-Error-Credibility-Interval.aspx (2021)."},{"key":"4198_CR21","unstructured":"The Delphi Group at Carnegie Mellon University in partnership with Facebook. Topline Report on COVID-19 Vaccination in the United States (2021)."},{"key":"4198_CR22","doi-asserted-by":"publisher","first-page":"1819","DOI":"10.1016\/S0140-6736(21)00947-8","volume":"397","author":"EJ Haas","year":"2021","unstructured":"Haas, E. J. et al. Impact and effectiveness of mRNA BNT162b2 vaccine against SARS-CoV-2 infections and COVID-19 cases, hospitalisations, and deaths following a nationwide vaccination campaign in Israel: an observational study using national surveillance data. Lancet 397, 1819\u20131829 (2021).","journal-title":"Lancet"},{"key":"4198_CR23","unstructured":"Kish, L. Survey Sampling (Wiley, 1965)."},{"key":"4198_CR24","unstructured":"Institute for Health Metrics and Evaluation (IHME). COVID-19 vaccine hesitancy. https:\/\/vaccine-hesitancy.healthdata.org\/ (2021)."},{"key":"4198_CR25","doi-asserted-by":"publisher","unstructured":"King, W. C., Rubinstein, M., Reinhart, A. & Mejia, R. J. Time trends and factors related to COVID-19 vaccine hesitancy from january-may 2021 among US adults: findings from a large-scale national survey. Preprint at https:\/\/doi.org\/10.1101\/2021.07.20.21260795 (2021).","DOI":"10.1101\/2021.07.20.21260795"},{"key":"4198_CR26","unstructured":"CDC. Estimates of vaccine hesitancy for COVID-19 (2021)."},{"key":"4198_CR27","unstructured":"Groves, R. M. et al. Survey Methodology Vol. 561 (Wiley, 2011)."},{"key":"4198_CR28","unstructured":"Dempsey, W. The hypothesis of testing: paradoxes arising out of reported coronavirus case-counts. Preprint at https:\/\/arxiv.org\/abs\/2005.10425 (2020)."},{"key":"4198_CR29","doi-asserted-by":"publisher","unstructured":"Isakov, M. & Kuriwaki, S. Towards principled unskewing: viewing 2020 election polls through a corrective lens from 2016. Harvard Data Science Review 2 https:\/\/doi.org\/10.1162\/99608f92.86a46f38 (2020).","DOI":"10.1162\/99608f92.86a46f38"},{"key":"4198_CR30","doi-asserted-by":"publisher","first-page":"270","DOI":"10.1038\/174270a0","volume":"174","author":"HO Hartley","year":"1954","unstructured":"Hartley, H. O. & Ross, A. Unbiased ratio estimators. Nature 174, 270\u2013271 (1954).","journal-title":"Nature"},{"key":"4198_CR31","doi-asserted-by":"publisher","unstructured":"Tiu, A., Susswein, Z., Merritt, A. & Bansal, S. Characterizing the spatiotemporal heterogeneity of the COVID-19 vaccination landscape. Preprint at https:\/\/doi.org\/10.1101\/2021.10.04.21263345 (2021).","DOI":"10.1101\/2021.10.04.21263345"},{"key":"4198_CR32","first-page":"173","volume":"28","author":"J Groen","year":"2012","unstructured":"Groen, J. Sources of error in survey and administrative data: the importance of reporting procedures. J. Off. Stat. 28, 173\u2013198 (2012).","journal-title":"J. Off. Stat."},{"key":"4198_CR33","doi-asserted-by":"crossref","first-page":"26","DOI":"10.1080\/01621459.1993.10594285","volume":"88","author":"XM Tu","year":"1993","unstructured":"Tu, X. M., Meng, X.-L. & Pagano, M. The AIDS epidemic: estimating survival after AIDS diagnosis from surveillance data. J. Am. Stat. Assoc. 88, 26\u201336 (1993).","journal-title":"J. Am. Stat. Assoc."},{"key":"4198_CR34","unstructured":"Barnes, O. & Burn-Murdoch, J. COVID response hampered by population data glitches. Financial Times (11 October 2021)."},{"key":"4198_CR35","unstructured":"Kennedy, C. et al. Evaluating online nonprobability surveys. Pew Research Center https:\/\/www.pewresearch.org\/methods\/2016\/05\/02\/evaluating-online-nonprobability-surveys\/ (2016)."},{"key":"4198_CR36","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1093\/poq\/nfx047","volume":"82","author":"C Kennedy","year":"2018","unstructured":"Kennedy, C. et al. An evaluation of the 2016 election polls in the United States. Public Opin. Q. 82, 1\u201333 (2018).","journal-title":"Public Opin. Q."},{"key":"4198_CR37","unstructured":"Auxier, B. & Anderson, M. Social media use in 2021. Pew Research Center https:\/\/www.pewresearch.org\/internet\/2021\/04\/07\/social-media-use-in-2021\/ (2021)."},{"key":"4198_CR38","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0249596","volume":"16","author":"SK Gadarian","year":"2021","unstructured":"Gadarian, S. K., Goodman, S. W. & Pepinsky, T. B. Partisanship, health behavior, and policy attitudes in the early stages of the COVID-19 pandemic. PLoS ONE 16, e0249596 (2021).","journal-title":"PLoS ONE"},{"key":"4198_CR39","unstructured":"Mercer, A., Lau, A. & Kennedy, C. For weighting online opt-in samples, what matters most? Pew Research Center https:\/\/www.pewresearch.org\/methods\/2018\/01\/26\/for-weighting-online-opt-in-samples-what-matters-most\/ (2018)."},{"key":"4198_CR40","unstructured":"Ryan, C. Computer and Internet Use in the United States: 2016. American Community Survey Report No. ACS-39 (U.S. Census Bureau, 2017)."},{"key":"4198_CR41","doi-asserted-by":"crossref","unstructured":"Biemer, P. P. & Lyberg, L. E. Introduction to Survey Quality (Wiley, 2003).","DOI":"10.1002\/0471458740"},{"key":"4198_CR42","unstructured":"Scheuren, F. What is a Survey? (American Statistical Association, 2004)."},{"key":"4198_CR43","unstructured":"Sukhatme, P. V. Sampling Theory of Surveys with Applications (1954)."},{"key":"4198_CR44","doi-asserted-by":"publisher","first-page":"558","DOI":"10.2307\/2342192","volume":"97","author":"J Neyman","year":"1934","unstructured":"Neyman, J. On the two different aspects of the representative method: the method of stratified sampling and the method of purposive selection. J. R. Stat. Soc. 97, 558\u2013625 (1934).","journal-title":"J. R. Stat. Soc."},{"key":"4198_CR45","doi-asserted-by":"publisher","first-page":"646","DOI":"10.1093\/poq\/nfl033","volume":"70","author":"RM Groves","year":"2006","unstructured":"Groves, R. M. Nonresponse rates and nonresponse bias in household surveys. Public Opin. Q. 70, 646\u2013675 (2006).","journal-title":"Public Opin. Q."},{"key":"4198_CR46","doi-asserted-by":"publisher","first-page":"1203","DOI":"10.1126\/science.1248506","volume":"343","author":"D Lazer","year":"2014","unstructured":"Lazer, D., Kennedy, R., King, G. & Vespignani, A. The parable of Google Flu: traps in big data analysis. Science 343, 1203\u20131205 (2014).","journal-title":"Science"},{"key":"4198_CR47","doi-asserted-by":"publisher","unstructured":"Salomon, J. A. et al. The US COVID-19 Trends and Impact Survey, 2020-2021: continuous real-time measurement of COVID-19 symptoms, risks, protective behaviors, testing and vaccination. Preprint at https:\/\/doi.org\/10.1101\/2021.07.24.21261076 (2021).","DOI":"10.1101\/2021.07.24.21261076"},{"key":"4198_CR48","doi-asserted-by":"publisher","first-page":"375","DOI":"10.1093\/pan\/mph024","volume":"12","author":"DK Park","year":"2004","unstructured":"Park, D. K., Gelman, A. & Bafumi, J. Bayesian multilevel estimation with poststratification: State-level estimates from national polls. Polit. Anal. 12, 375\u2013385 (2004).","journal-title":"Polit. Anal."},{"key":"4198_CR49","unstructured":"Associated Press\u2013NORC Center for Public Affairs Research. The June 2021 AP\u2013NORC center poll (2021)."},{"key":"4198_CR50","doi-asserted-by":"publisher","first-page":"980","DOI":"10.1016\/j.ijforecast.2014.06.001","volume":"31","author":"W Wang","year":"2015","unstructured":"Wang, W., Rothschild, D., Goel, S. & Gelman, A. Forecasting elections with non-representative polls. Int. J. Forecast. 31, 980\u2013991 (2015).","journal-title":"Int. J. Forecast."},{"key":"4198_CR51","doi-asserted-by":"publisher","first-page":"249","DOI":"10.1214\/16-STS598","volume":"32","author":"MR Elliott","year":"2017","unstructured":"Elliott, M. R. & Valliant, R. Inference for nonprobability samples. Stat. Sci. 32, 249\u2013264 (2017).","journal-title":"Stat. Sci."},{"key":"4198_CR52","doi-asserted-by":"publisher","first-page":"932","DOI":"10.1093\/jssam\/smz023","volume":"8","author":"RJ Little","year":"2020","unstructured":"Little, R. J., West, B. T., Boonstra, P. S. & Hu, J. Measures of the degree of departure from ignorable sample selection. J. Surv. Stat. Methodol. 8, 932\u2013964 (2020).","journal-title":"J. Surv. Stat. Methodol."},{"key":"4198_CR53","doi-asserted-by":"publisher","first-page":"120","DOI":"10.1093\/jssam\/smz051","volume":"8","author":"A Wi\u015bniowski","year":"2020","unstructured":"Wi\u015bniowski, A., Sakshaug, J. W., Perez Ruiz, D. A. & Blom, A. G. Integrating probability and nonprobability samples for survey inference. J. Surv. Stat. Methodol.8, 120\u2013147 (2020).","journal-title":"J. Surv. Stat. Methodol."},{"key":"4198_CR54","doi-asserted-by":"publisher","first-page":"445","DOI":"10.1111\/rssb.12354","volume":"82","author":"S Yang","year":"2020","unstructured":"Yang, S., Kim, J. K. & Song, R. Doubly robust inference when combining probability and non-probability samples with high dimensional data. J. R. Stat. Soc. B 82, 445\u2013465 (2020).","journal-title":"J. R. Stat. Soc. B"},{"key":"4198_CR55","unstructured":"U.S. Census Bureau. Methodology for the United States population estimates: Vintage 2019."},{"key":"4198_CR56","unstructured":"Bethlehem, J. in Survey Nonresponse (eds Groves, R. M. et al.) 275\u2013288 (Wiley, 2002)."},{"key":"4198_CR57","unstructured":"Meng, X.-L. in Past, Present, and Future of Statistical Science (eds Lin, X. et al.) 537\u2013562 (CRC Press, 2014)."},{"key":"4198_CR58","doi-asserted-by":"publisher","first-page":"218","DOI":"10.1080\/07474938.2013.808567","volume":"33","author":"X-L Meng","year":"2014","unstructured":"Meng, X.-L. & Xie, X. I got more data, my model is more refined, but my estimator is getting worse! Am I just dumb? Econom. Rev. 33, 218\u2013250 (2014).","journal-title":"Econom. Rev."},{"key":"4198_CR59","doi-asserted-by":"crossref","unstructured":"Heckman, J. J. Sample selection bias as a specification error. Econometrica 153\u2013161 (1979).","DOI":"10.2307\/1912352"},{"key":"4198_CR60","unstructured":"CDC. Reporting COVID-19 vaccination demographic data. https:\/\/www.cdc.gov\/coronavirus\/2019-ncov\/vaccines\/distributing\/demographics-vaccination-data.html (2021)."}],"container-title":["Nature"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41586-021-04198-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41586-021-04198-4","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41586-021-04198-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,13]],"date-time":"2024-09-13T20:57:16Z","timestamp":1726261036000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41586-021-04198-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,12,8]]},"references-count":60,"journal-issue":{"issue":"7890","published-print":{"date-parts":[[2021,12,23]]}},"alternative-id":["4198"],"URL":"https:\/\/doi.org\/10.1038\/s41586-021-04198-4","relation":{"has-preprint":[{"id-type":"doi","id":"10.1101\/2021.06.10.21258694","asserted-by":"object"}]},"ISSN":["0028-0836","1476-4687"],"issn-type":[{"value":"0028-0836","type":"print"},{"value":"1476-4687","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,12,8]]},"assertion":[{"value":"18 June 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"29 October 2021","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"8 December 2021","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}