{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,22]],"date-time":"2026-04-22T16:53:19Z","timestamp":1776876799033,"version":"3.51.2"},"reference-count":96,"publisher":"American Chemical Society (ACS)","issue":"13","license":[{"start":{"date-parts":[[2023,6,20]],"date-time":"2023-06-20T00:00:00Z","timestamp":1687219200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100002428","name":"Austrian Science Fund","doi-asserted-by":"publisher","award":["J-4415"],"award-info":[{"award-number":["J-4415"]}],"id":[{"id":"10.13039\/501100002428","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Machine Learning for Pharmaceutical Discovery and Synthesis Consortium"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["J. Chem. Inf. Model."],"published-print":{"date-parts":[[2023,7,10]]},"DOI":"10.1021\/acs.jcim.3c00373","type":"journal-article","created":{"date-parts":[[2023,6,20]],"date-time":"2023-06-20T09:04:46Z","timestamp":1687251886000},"page":"4012-4029","source":"Crossref","is-referenced-by-count":66,"title":["Characterizing Uncertainty in Machine Learning for Chemistry"],"prefix":"10.1021","volume":"63","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8404-6596","authenticated-orcid":true,"given":"Esther","family":"Heid","sequence":"first","affiliation":[{"name":"Department of Chemical Engineering, Massachusetts Institute of Technology, Cambridge, Massachusetts 02139, United States"},{"name":"Institute of Materials Chemistry, TU Wien, 1060 Vienna, Austria"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2704-7717","authenticated-orcid":true,"given":"Charles J.","family":"McGill","sequence":"additional","affiliation":[{"name":"Department of Chemical Engineering, Massachusetts Institute of Technology, Cambridge, Massachusetts 02139, United States"},{"name":"Department of Chemical and Life Science Engineering, Virginia Commonwealth University, Richmond, Virginia 23284, United States"}]},{"given":"Florence H.","family":"Vermeire","sequence":"additional","affiliation":[{"name":"Department of Chemical Engineering, Massachusetts Institute of Technology, Cambridge, Massachusetts 02139, United States"},{"name":"Department of Chemical Engineering, KU Leuven, Celestijnenlaan 200F, B-3001 Leuven, Belgium"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2603-9694","authenticated-orcid":true,"given":"William H.","family":"Green","sequence":"additional","affiliation":[{"name":"Department of Chemical Engineering, Massachusetts Institute of Technology, Cambridge, Massachusetts 02139, United States"}]}],"member":"316","published-online":{"date-parts":[[2023,6,20]]},"reference":[{"key":"ref1\/cit1","unstructured":"Gilmer, J.; Schoenholz, S. S.; Riley, P. F.; Vinyals, O.; Dahl, G. E. Neural Message Passing for Quantum Chemistry.\n                      ICML'17: Proceedings of the 34th International Conference on Machine Learning - Volume 70\n                      ; 2017; pp 1263\u20131272."},{"key":"ref2\/cit2","unstructured":"Klicpera, J.; Gro\u00df, J.; G\u00fcnnemann, S. Directional Message Passing for Molecular Graphs. 2020, arXiv:2003.03123.\n                      arXiv preprint\n                      . https:\/\/arxiv.org\/abs\/2003.03123 (accessed 2023-06-12)."},{"key":"ref3\/cit3","unstructured":"Zhang, S.; Liu, Y.; Xie, L. Molecular Mechanics-Driven Graph Neural Network with Multiplex Graph for Molecular Structures. 2020, arXiv:2011.07457.\n                      arXiv preprint\n                      . https:\/\/arxiv.org\/abs\/2011.07457 (accessed 2023-06-12)."},{"key":"ref4\/cit4","unstructured":"Alperstein, Z.; Cherkasov, A.; Rolfe, J. T. All Smiles Variational Autoencoder. 2019, arXiv:1905.13343.\n                      arXiv preprint\n                      . https:\/\/arxiv.org\/abs\/1905.13343 (accessed 2023-06-12)."},{"key":"ref5\/cit5","doi-asserted-by":"publisher","DOI":"10.1016\/j.comtox.2019.01.001"},{"key":"ref6\/cit6","doi-asserted-by":"publisher","DOI":"10.1093\/bib\/bbaa266"},{"key":"ref7\/cit7","doi-asserted-by":"publisher","DOI":"10.1021\/acs.jcim.9b00410"},{"key":"ref8\/cit8","doi-asserted-by":"publisher","DOI":"10.1021\/acs.jcim.9b00237"},{"key":"ref9\/cit9","doi-asserted-by":"publisher","DOI":"10.1039\/C7SC02664A"},{"key":"ref10\/cit10","doi-asserted-by":"publisher","DOI":"10.1021\/acs.iecr.2c00442"},{"key":"ref11\/cit11","doi-asserted-by":"publisher","DOI":"10.1021\/jacs.2c01768"},{"key":"ref12\/cit12","doi-asserted-by":"publisher","DOI":"10.1126\/science.aar5169"},{"key":"ref13\/cit13","doi-asserted-by":"publisher","DOI":"10.1016\/j.chempr.2020.02.017"},{"key":"ref14\/cit14","doi-asserted-by":"publisher","DOI":"10.1088\/2632-2153\/abc81d"},{"key":"ref15\/cit15","doi-asserted-by":"publisher","DOI":"10.1007\/s10562-019-02705-x"},{"key":"ref16\/cit16","doi-asserted-by":"publisher","DOI":"10.1039\/D0SC04896H"},{"key":"ref17\/cit17","doi-asserted-by":"publisher","DOI":"10.1021\/acs.jpca.0c05992"},{"key":"ref18\/cit18","doi-asserted-by":"publisher","DOI":"10.1126\/science.aau5631"},{"key":"ref19\/cit19","doi-asserted-by":"publisher","DOI":"10.1021\/acs.jcim.1c00975"},{"key":"ref20\/cit20","doi-asserted-by":"publisher","DOI":"10.1038\/nature25978"},{"key":"ref21\/cit21","doi-asserted-by":"crossref","unstructured":"Schwaller, P.; Petraglia, R.; Zullo, V.; Nair, V. H.; Haeuselmann, R. A.; Pisoni, R.; Bekas, C.; Iuliano, A.; Laino, T. Predicting Retrosynthetic Pathways Using a Combined Linguistic Model and Hyper-Graph Exploration Strategy. 2019, arXiv:1910.08036.\n                      arXiv preprint\n                      . https:\/\/arxiv.org\/abs\/1910.08036 (accessed 2023-06-12).","DOI":"10.26434\/chemrxiv.9992489"},{"key":"ref22\/cit22","doi-asserted-by":"publisher","DOI":"10.1039\/C9CC05122H"},{"key":"ref23\/cit23","unstructured":"Chen, B.; Shen, T.; Jaakkola, T. S.; Barzilay, R. Learning to Make Generalizable and Diverse Predictions for Retrosynthesis. 2019, arXiv:1910.09688.\n                      arXiv preprint\n                      . https:\/\/arxiv.org\/abs\/1910.09688 (accessed 2023-06-12)."},{"key":"ref24\/cit24","doi-asserted-by":"publisher","DOI":"10.1021\/acs.jcim.1c01192"},{"key":"ref25\/cit25","doi-asserted-by":"publisher","DOI":"10.1039\/C7ME00107J"},{"key":"ref26\/cit26","doi-asserted-by":"publisher","DOI":"10.1021\/acscentsci.9b00576"},{"key":"ref27\/cit27","doi-asserted-by":"publisher","DOI":"10.1021\/acscentsci.7b00303"},{"key":"ref28\/cit28","doi-asserted-by":"publisher","DOI":"10.1039\/C8SC04228D"},{"key":"ref29\/cit29","doi-asserted-by":"publisher","DOI":"10.1089\/big.2016.0051"},{"key":"ref30\/cit30","unstructured":"Papernot, N.; McDaniel, P. Deep k-Nearest Neighbors: Towards Confident, Interpretable and Robust Deep Learning. 2018, arXiv:1803.04765.\n                      arXiv preprint\n                      . https:\/\/arxiv.org\/abs\/1803.04765 (accessed 2023-06-12)."},{"key":"ref31\/cit31","doi-asserted-by":"publisher","DOI":"10.1016\/S0951-8320(96)00077-4"},{"key":"ref32\/cit32","doi-asserted-by":"publisher","DOI":"10.1016\/j.strusafe.2008.06.020"},{"key":"ref33\/cit33","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2013.07.030"},{"key":"ref34\/cit34","unstructured":"Kendall, A.; Gal, Y. What Uncertainties Do We Need in Bayesian Deep Learning for Computer Vision?\n                      NIPS'17: Proceedings of the 31st International Conference on Neural Information Processing Systems\n                      ; 2017; p 5580."},{"key":"ref35\/cit35","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-021-05946-3"},{"key":"ref36\/cit36","doi-asserted-by":"publisher","DOI":"10.1162\/neco.1992.4.1.1"},{"key":"ref37\/cit37","series-title":"Synthesis Lectures on Artificial Intelligence and Machine Learning","doi-asserted-by":"crossref","DOI":"10.1007\/978-3-031-01560-1","volume-title":"Active Learning","author":"Settles B.","year":"2012"},{"key":"ref38\/cit38","doi-asserted-by":"crossref","unstructured":"Nix, D. A.; Weigend, A. S. Estimating the Mean and Variance of the Target Probability Distribution.\n                      Proceedings of 1994 ieee international conference on neural networks (ICNN\u201994)\n                      ; 1994; pp 55\u201360.","DOI":"10.1109\/ICNN.1994.374138"},{"key":"ref39\/cit39","volume-title":"Bayesian Learning for Neural Networks","volume":"118","author":"Neal R. M.","year":"2012"},{"key":"ref40\/cit40","unstructured":"Lakshminarayanan, B.; Pritzel, A.; Blundell, C. Simple and Scalable Predictive Uncertainty Estimation Using Deep Ensembles.\n                      31st Conference on Neural Information Processing Systems (NIPS 2017)\n                      ; 2017; pp 6402\u20136413."},{"key":"ref41\/cit41","doi-asserted-by":"crossref","first-page":"569","DOI":"10.1007\/978-1-4612-4380-9_41","volume-title":"Breakthroughs in statistics","author":"Efron B.","year":"1992"},{"key":"ref42\/cit42","unstructured":"Gal, Y.; Ghahramani, Z. Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning.\n                      Proceedings of The 33rd International Conference on Machine Learning, PMLR\n                      ; 2016; pp 1050\u20131059."},{"key":"ref43\/cit43","doi-asserted-by":"publisher","DOI":"10.1021\/acs.jcim.9b00297"},{"key":"ref44\/cit44","doi-asserted-by":"publisher","DOI":"10.1021\/acscentsci.1c00546"},{"key":"ref45\/cit45","first-page":"371","volume":"9","author":"Shafer G.","year":"2008","journal-title":"J. Mach. Learn. Res."},{"key":"ref46\/cit46","doi-asserted-by":"publisher","DOI":"10.1021\/acs.jcim.0c00502"},{"key":"ref47\/cit47","doi-asserted-by":"publisher","DOI":"10.1021\/acs.jcim.9b00975"},{"key":"ref48\/cit48","doi-asserted-by":"publisher","DOI":"10.1063\/1.1744539"},{"key":"ref49\/cit49","doi-asserted-by":"publisher","DOI":"10.1038\/sdata.2014.22"},{"key":"ref50\/cit50","first-page":"22118","volume":"33","author":"Hu W.","year":"2020","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref51\/cit51","doi-asserted-by":"publisher","DOI":"10.1021\/acs.jcim.7b00083"},{"key":"ref52\/cit52","doi-asserted-by":"publisher","DOI":"10.1002\/anie.200462457"},{"key":"ref53\/cit53","doi-asserted-by":"publisher","DOI":"10.1021\/ci600423u"},{"key":"ref54\/cit54","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.7626488"},{"key":"ref55\/cit55","doi-asserted-by":"publisher","DOI":"10.1021\/ci300415d"},{"key":"ref56\/cit56","unstructured":"Chemprop: Message Passing Neural Networks for Molecule\nProperty Prediction,\nv1.4.1\n                      . https:\/\/github.com\/chemprop\/chemprop (accessed 2022-03-30)."},{"key":"ref57\/cit57","doi-asserted-by":"publisher","DOI":"10.1063\/1.5019779"},{"key":"ref58\/cit58","doi-asserted-by":"publisher","DOI":"10.1021\/c160017a018"},{"key":"ref59\/cit59","unstructured":"Landrum, G.\n                      RDKit: Open-source Cheminformatics\n                      . 2006. https:\/\/www.rdkit.org\/ (accessed 2023-06-12)."},{"key":"ref60\/cit60","unstructured":"McGill, C.; Green, W.\n                      Ensemble Projection\n                      . https:\/\/github.com\/cjmcgill\/ensemble_projection (accessed 2023-03-08)."},{"key":"ref61\/cit61","unstructured":"SchNetPack - Deep Neural\nNetworks for Atomistic Systems, v1.0.0\n                      . https:\/\/github.com\/atomistic-machine-learning\/schnetpack (accessed\n2022-03-30)."},{"key":"ref62\/cit62","unstructured":"McGill, C.; Green, W.\n                      Ensemble Projection\n                      . https:\/\/github.com\/cjmcgill\/ensemble_projection (accessed 2023-03-08)."},{"key":"ref63\/cit63","doi-asserted-by":"crossref","unstructured":"Heid, E.; McGill, C.; Vermeire, F.; Green, W.\n                      Characterizing Uncertainty in Machine Learning for Chemistry Scripts\n                      . https:\/\/github.com\/cjmcgill\/characterizing_uncertainty_scripts (accessed 2023-03-08).","DOI":"10.26434\/chemrxiv-2023-00vcg-v2"},{"key":"ref64\/cit64","unstructured":"Xie, T.; France-Lanord, A.; Wang, Y.; Lopez, J.; Stolberg, M. A.; Hill, M.; Leverick, G. M.; Gomez-Bombarelli, R.; Johnson, J. A.; Shao-Horn, Y.  Accelerating the Screening of Amorphous Polymer Electrolytes by Learning to Reduce Random and Systematic Errors in Molecular Dynamics Simulations. 2021, arXiv:2101.05339.\n                      arXiv preprint\n                      . https:\/\/arxiv.org\/abs\/2101.05339 (accessed 2023-06-12)."},{"key":"ref65\/cit65","doi-asserted-by":"crossref","unstructured":"Le, Q. V.; Smola, A. J.; Canu, S. Heteroscedastic Gaussian Process Regression.\n                      Proceedings of the 22nd international conference on Machine learning\n                      ; 2005; pp 489\u2013496.","DOI":"10.1145\/1102351.1102413"},{"key":"ref66\/cit66","unstructured":"Chua, K.; Calandra, R.; McAllister, R.; Levine, S. Deep Reinforcement Learning in a Handful of Trials Using Probabilistic Dynamics Models.\n                      32nd Conference on Neural Information Processing Systems (NeurIPS 2018)\n                      ; 2018, Vol. 31."},{"key":"ref67\/cit67","unstructured":"Seitzer, M.; Tavakoli, A.; Antic, D.; Martius, G. On the Pitfalls of Heteroscedastic Uncertainty Estimation with Probabilistic Neural Networks. 2022, arXiv:2203.09168.\n                      arXiv preprint\n                      . https:\/\/arxiv.org\/abs\/2203.09168 (accessed 2023-06-12)."},{"key":"ref68\/cit68","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-44677-6_67"},{"key":"ref69\/cit69","doi-asserted-by":"publisher","DOI":"10.1002\/qua.24912"},{"key":"ref70\/cit70","doi-asserted-by":"publisher","DOI":"10.1021\/acs.jcim.1c01103"},{"key":"ref71\/cit71","doi-asserted-by":"publisher","DOI":"10.1039\/D1SC05677H"},{"key":"ref72\/cit72","doi-asserted-by":"publisher","DOI":"10.1038\/s41591-018-0107-6"},{"key":"ref73\/cit73","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-019-1390-1"},{"key":"ref74\/cit74","doi-asserted-by":"publisher","DOI":"10.1007\/s10822-014-9747-x"},{"key":"ref75\/cit75","doi-asserted-by":"publisher","DOI":"10.1021\/ci034243x"},{"key":"ref76\/cit76","doi-asserted-by":"publisher","DOI":"10.1007\/s10822-017-0044-3"},{"key":"ref77\/cit77","doi-asserted-by":"publisher","DOI":"10.1080\/1062936X.2021.1883107"},{"key":"ref78\/cit78","doi-asserted-by":"publisher","DOI":"10.1021\/acs.jpclett.0c00500"},{"key":"ref79\/cit79","doi-asserted-by":"publisher","DOI":"10.1021\/jm9602928"},{"key":"ref80\/cit80","doi-asserted-by":"publisher","DOI":"10.1021\/ci400084k"},{"key":"ref81\/cit81","doi-asserted-by":"publisher","DOI":"10.1186\/s13321-021-00576-2"},{"key":"ref82\/cit82","doi-asserted-by":"publisher","DOI":"10.1021\/acs.jcim.0c00416"},{"key":"ref83\/cit83","doi-asserted-by":"publisher","DOI":"10.4155\/fmc-2020-0259"},{"key":"ref84\/cit84","unstructured":"Magar, R.; Wang, Y.; Lorsung, C.; Liang, C.; Ramasubramanian, H.; Li, P.; Farimani, A. B. AugLiChem: Data Augmentation Library of Chemical Structures for Machine Learning. 2021, arXiv:2111.15112.\n                      arXiv preprint\n                      . https:\/\/arxiv.org\/abs\/2111.15112 (accessed 2023-06-12)."},{"key":"ref85\/cit85","doi-asserted-by":"publisher","DOI":"10.1039\/C8ME00012C"},{"key":"ref86\/cit86","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-021-21895-w"},{"key":"ref87\/cit87","unstructured":"Li, J.; Cai, D.; He, X. Learning Graph-level Representation for Drug Discovery. 2017, arXiv:1709.03741.\n                      arXiv preprint\n                      . https:\/\/arxiv.org\/abs\/1709.03741 (accessed 2023-06-12)."},{"key":"ref88\/cit88","doi-asserted-by":"publisher","DOI":"10.1186\/s13321-020-00430-x"},{"key":"ref89\/cit89","doi-asserted-by":"crossref","unstructured":"Bai, P.; Miljkovi\u0107, F.; Ge, Y.; Greene, N.; John, B.; Lu, H. Hierarchical Clustering Split for Low-Bias Evaluation of Drug-Target Interaction Prediction.\n                      2021 IEEE International Conference on Bioinformatics and Biomedicine (BIBM)\n                      ; 2021; pp 641\u2013644.","DOI":"10.1109\/BIBM52615.2021.9669515"},{"key":"ref90\/cit90","doi-asserted-by":"publisher","DOI":"10.1039\/C9SC00616H"},{"key":"ref91\/cit91","doi-asserted-by":"publisher","DOI":"10.1016\/j.cej.2021.129307"},{"key":"ref92\/cit92","unstructured":"Kuleshov, V.; Fenner, N.; Ermon, S. Accurate Uncertainties for Deep Learning Using Calibrated Regression.\n                      Proceedings of the 35 th International Conference on Machine Learning\n                      ; 2018; pp 2796\u20132804."},{"key":"ref93\/cit93","doi-asserted-by":"publisher","DOI":"10.3390\/s22155540"},{"key":"ref94\/cit94","unstructured":"Zelikman, E.; Healy, C.; Zhou, S.; Avati, A. CRUDE: Calibrating Regression Uncertainty Distributions Empirically. 2021, arXiv:2005.12496 [cs, stat].\n                      arXiv preprint\n                      . https:\/\/arxiv.org\/abs\/2005.12496 (accessed 2023-06-12)."},{"key":"ref95\/cit95","doi-asserted-by":"publisher","DOI":"10.1088\/2632-2153\/ab7e1a"},{"key":"ref96\/cit96","doi-asserted-by":"crossref","unstructured":"Reuther, A.  Interactive Supercomputing on 40,000 Cores for Machine Learning and Data Analysis.\n                      2018 IEEE High Performance extreme Computing Conference (HPEC)\n                      ; 2018; pp 1\u20136.","DOI":"10.1109\/HPEC.2018.8547629"}],"container-title":["Journal of Chemical Information and Modeling"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/pubs.acs.org\/doi\/pdf\/10.1021\/acs.jcim.3c00373","content-type":"application\/pdf","content-version":"vor","intended-application":"unspecified"},{"URL":"https:\/\/pubs.acs.org\/doi\/pdf\/10.1021\/acs.jcim.3c00373","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,7,10]],"date-time":"2023-07-10T04:09:42Z","timestamp":1688962182000},"score":1,"resource":{"primary":{"URL":"https:\/\/pubs.acs.org\/doi\/10.1021\/acs.jcim.3c00373"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,6,20]]},"references-count":96,"journal-issue":{"issue":"13","published-print":{"date-parts":[[2023,7,10]]}},"alternative-id":["10.1021\/acs.jcim.3c00373"],"URL":"https:\/\/doi.org\/10.1021\/acs.jcim.3c00373","relation":{"has-preprint":[{"id-type":"doi","id":"10.26434\/chemrxiv-2023-00vcg","asserted-by":"object"},{"id-type":"doi","id":"10.26434\/chemrxiv-2023-00vcg-v3","asserted-by":"object"},{"id-type":"doi","id":"10.26434\/chemrxiv-2023-00vcg-v2","asserted-by":"object"}]},"ISSN":["1549-9596","1549-960X"],"issn-type":[{"value":"1549-9596","type":"print"},{"value":"1549-960X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,6,20]]}}}