{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,23]],"date-time":"2026-10-23T12:52:32Z","timestamp":1792759952612,"version":"4.5.0"},"reference-count":32,"publisher":"National Academy of Sciences","issue":"43","license":[{"start":{"date-parts":[[2026,10,19]],"date-time":"2026-10-19T00:00:00Z","timestamp":1792368000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100003977","name":"Israel Science Foundation","doi-asserted-by":"publisher","award":["1324\/23"],"award-info":[{"award-number":["1324\/23"]}],"id":[{"id":"10.13039\/501100003977","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Manifold Community Fund","award":["NA"],"award-info":[{"award-number":["NA"]}]},{"name":"EA Community Choice","award":["NA"],"award-info":[{"award-number":["NA"]}]}],"content-domain":{"domain":["www.pnas.org"],"crossmark-restriction":true},"short-container-title":["Proc. Natl. Acad. Sci. U.S.A."],"published-print":{"date-parts":[[2026,10,27]]},"abstract":"<jats:p>\n                    Using approximately one million resolved contracts and questions across four collective forecasting platforms, twelve domains, and time horizons ranging from hours to years, we evaluate the calibration and accuracy of collective forecasts in real-world settings. We found that forecast probabilities were overall highly calibrated: events assigned a probability of\n                    <jats:italic toggle=\"yes\">p<\/jats:italic>\n                    % occurred approximately\n                    <jats:italic toggle=\"yes\">p<\/jats:italic>\n                    % of the time. This broad pattern was evident across platforms and many domains, although fine-grained analyses revealed systematic tendencies toward over- or underestimation in highly consequential areas. While this does not mean that forecasting platforms provide certainty about individual events, our findings suggest that large-scale aggregation of judgments contains meaningful information about real-world uncertainty; when examined in aggregate, the predictions generated by the platforms provided informative estimates of event likelihood.\n                  <\/jats:p>","DOI":"10.1073\/pnas.2615627123","type":"journal-article","created":{"date-parts":[[2026,10,19]],"date-time":"2026-10-19T19:01:49Z","timestamp":1792436509000},"update-policy":"https:\/\/doi.org\/10.1073\/pnas.cm10313","source":"Crossref","is-referenced-by-count":0,"title":["Calibration, accuracy, and domain variation across one million collective forecasts from prediction platforms"],"prefix":"10.1073","volume":"123","author":[{"ORCID":"https:\/\/orcid.org\/0009-0006-8587-9596","authenticated-orcid":false,"given":"Justin","family":"Dickerson","sequence":"first","affiliation":[{"name":"Private address","place":["Indianapolis"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9717-8881","authenticated-orcid":false,"given":"Hilla","family":"Shinitzky","sequence":"additional","affiliation":[{"id":[{"id":"https:\/\/ror.org\/04mhzgx49","id-type":"ROR","asserted-by":"publisher"}],"name":"The School of Psychological Science, Tel Aviv University","place":["Tel Aviv, Israel"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-1609-3872","authenticated-orcid":false,"given":"Maayan","family":"Peer","sequence":"additional","affiliation":[{"id":[{"id":"https:\/\/ror.org\/04mhzgx49","id-type":"ROR","asserted-by":"publisher"}],"name":"The School of Psychological Science, Tel Aviv University","place":["Tel Aviv, Israel"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4287-4801","authenticated-orcid":false,"given":"Mattan S.","family":"Ben-Shachar","sequence":"additional","affiliation":[{"id":[{"id":"https:\/\/ror.org\/04mhzgx49","id-type":"ROR","asserted-by":"publisher"}],"name":"The School of Psychological Science, Tel Aviv University","place":["Tel Aviv, Israel"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Michael","family":"Gilead","sequence":"additional","affiliation":[{"id":[{"id":"https:\/\/ror.org\/04mhzgx49","id-type":"ROR","asserted-by":"publisher"}],"name":"The School of Psychological Science, Tel Aviv University","place":["Tel Aviv, Israel"]}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"341","published-online":{"date-parts":[[2026,10,19]]},"reference":[{"key":"e_1_3_4_1_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.185.4157.1124"},{"key":"e_1_3_4_2_2","doi-asserted-by":"publisher","DOI":"10.21236\/ADA101986"},{"key":"e_1_3_4_3_2","doi-asserted-by":"publisher","DOI":"10.1007\/BF00122574"},{"key":"e_1_3_4_4_2","doi-asserted-by":"publisher","DOI":"10.1038\/075450a0"},{"key":"e_1_3_4_5_2","volume-title":"The Wisdom of Crowds","author":"Surowiecki J.","year":"2005","unstructured":"J. Surowiecki, The Wisdom of Crowds (Anchor Books, New York, 2005)."},{"key":"e_1_3_4_6_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijforecast.2008.03.007"},{"key":"e_1_3_4_7_2","doi-asserted-by":"publisher","DOI":"10.1257\/0895330041371321"},{"key":"e_1_3_4_8_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.1157679"},{"key":"e_1_3_4_9_2","first-page":"657","volume-title":"Handbook of Economic Forecasting","author":"Snowberg E.","year":"2013","unstructured":"E. Snowberg, J. Wolfers, E. Zitzewitz, \u201cPrediction markets for economic forecasting\u201d in Handbook of Economic Forecasting (Elsevier, 2013), vol. 2, pp. 657\u2013687."},{"key":"e_1_3_4_10_2","doi-asserted-by":"publisher","DOI":"10.1093\/restud\/rdv014"},{"key":"e_1_3_4_11_2","doi-asserted-by":"publisher","DOI":"10.1111\/j.1468-0297.2012.02561.x"},{"key":"e_1_3_4_12_2","doi-asserted-by":"publisher","DOI":"10.1111\/poms.13175"},{"key":"e_1_3_4_13_2","doi-asserted-by":"crossref","unstructured":"J. D. Clinton T. Huang Prediction markets? The accuracy and efficiency of $2.4 billion in the 2024 presidential election. SocArXiv Center for Open Science (2025).","DOI":"10.31235\/osf.io\/d5yx2_v2"},{"key":"e_1_3_4_14_2","doi-asserted-by":"publisher","DOI":"10.1175\/1520-0493(1950)078<0001:VOFEIT>2.0.CO;2"},{"key":"e_1_3_4_15_2","volume-title":"Expert Political Judgment: How Good Is It? How Can We Know?","author":"Tetlock P. E.","year":"2005","unstructured":"P. E. Tetlock, Expert Political Judgment: How Good Is It? How Can We Know? (Princeton Univ. Press, Princeton, NJ, 2005)."},{"key":"e_1_3_4_16_2","doi-asserted-by":"publisher","DOI":"10.1175\/1520-0450(1973)012<0595:ANVPOT>2.0.CO;2"},{"key":"e_1_3_4_17_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.1157679"},{"key":"e_1_3_4_18_2","doi-asserted-by":"publisher","DOI":"10.1037\/0033-295X.101.4.547"},{"key":"e_1_3_4_19_2","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1099-0771(199712)10:4<293::AID-BDM256>3.0.CO;2-P"},{"key":"e_1_3_4_20_2","doi-asserted-by":"publisher","DOI":"10.1037\/0033-295X.104.2.406"},{"key":"e_1_3_4_21_2","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1099-0771(199906)12:2<141::AID-BDM312>3.0.CO;2-C"},{"key":"e_1_3_4_22_2","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.1050.0409"},{"key":"e_1_3_4_23_2","doi-asserted-by":"publisher","DOI":"10.1080\/02684527.2016.1147164"},{"key":"e_1_3_4_24_2","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.2020.3882"},{"key":"e_1_3_4_25_2","doi-asserted-by":"publisher","DOI":"10.1002\/ffo2.157"},{"key":"e_1_3_4_26_2","doi-asserted-by":"publisher","DOI":"10.1017\/S1930297500005647"},{"key":"e_1_3_4_27_2","unstructured":"J. Dickerson Project themis. GitHub. https:\/\/github.com\/wasabipesto\/themis. Deposited 11 December 2023."},{"key":"e_1_3_4_28_2","unstructured":"M. Kuhn H. Wickham Tidymodels: a collection of packages for modeling and machine learning using tidyverse principles (2020). https:\/\/www.tidymodels.org."},{"key":"e_1_3_4_29_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-24277-4"},{"key":"e_1_3_4_30_2","unstructured":"M. Kuhn D. Vaughan E. Ruiz Probably: Tools for Post-Processing Predicted Values (2024). https:\/\/github.com\/tidymodels\/probably."},{"key":"e_1_3_4_31_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.frl.2018.08.003"},{"key":"e_1_3_4_32_2","doi-asserted-by":"crossref","unstructured":"M. Kull T. M. Silva Filho P. Flach Beyond sigmoids: How to obtain well-calibrated probabilities from binary classifiers with beta calibration. Electron. J. Stat. 11 5052\u20135080 (2017).","DOI":"10.1214\/17-EJS1338SI"}],"container-title":["Proceedings of the National Academy of Sciences"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/pnas.org\/doi\/pdf\/10.1073\/pnas.2615627123","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,10,19]],"date-time":"2026-10-19T19:02:12Z","timestamp":1792436532000},"score":1,"resource":{"primary":{"URL":"https:\/\/pnas.org\/doi\/10.1073\/pnas.2615627123"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,10,19]]},"references-count":32,"journal-issue":{"issue":"43","published-print":{"date-parts":[[2026,10,27]]}},"alternative-id":["10.1073\/pnas.2615627123"],"URL":"https:\/\/doi.org\/10.1073\/pnas.2615627123","relation":{},"ISSN":["0027-8424","1091-6490"],"issn-type":[{"value":"0027-8424","type":"print"},{"value":"1091-6490","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,10,19]]},"assertion":[{"value":"2026-05-06","order":0,"name":"received","label":"Received","group":{"name":"publication_history","label":"Publication History"}},{"value":"2026-08-31","order":2,"name":"accepted","label":"Accepted","group":{"name":"publication_history","label":"Publication History"}},{"value":"2026-10-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}],"article-number":"e2615627123"}}