{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,8]],"date-time":"2026-10-08T19:12:39Z","timestamp":1791486759531,"version":"4.3.4"},"reference-count":33,"publisher":"Informa UK Limited","issue":"548","funder":[{"DOI":"10.13039\/501100000930","name":"National Science Foundation","doi-asserted-by":"publisher","award":["CCF-1553086"],"award-info":[{"award-number":["CCF-1553086"]}],"id":[{"id":"10.13039\/501100000930","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000930","name":"National Science Foundation","doi-asserted-by":"publisher","award":["HDR 1934578"],"award-info":[{"award-number":["HDR 1934578"]}],"id":[{"id":"10.13039\/501100000930","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000006","name":"Office of Naval Research","doi-asserted-by":"publisher","award":["N00014-19-2288"],"award-info":[{"award-number":["N00014-19-2288"]}],"id":[{"id":"10.13039\/100000006","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Stanford DAWN Consortium"}],"content-domain":{"domain":["www.tandfonline.com"],"crossmark-restriction":true},"short-container-title":["Journal of the American Statistical Association"],"published-print":{"date-parts":[[2024,10]]},"DOI":"10.1080\/01621459.2023.2298037","type":"journal-article","created":{"date-parts":[[2023,12,26]],"date-time":"2023-12-26T11:46:26Z","timestamp":1703591186000},"page":"3033-3044","update-policy":"https:\/\/doi.org\/10.1080\/tandf_crossmark_01","source":"Crossref","is-referenced-by-count":39,"title":["Robust Validation: Confident Predictions Even When Distributions Shift"],"prefix":"10.1080","volume":"119","author":[{"given":"Maxime","family":"Cauchois","sequence":"first","affiliation":[{"name":"Department of Statistics, Stanford University","place":["Stanford"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Suyash","family":"Gupta","sequence":"additional","affiliation":[{"name":"Department of Statistics, Stanford University","place":["Stanford"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alnur","family":"Ali","sequence":"additional","affiliation":[{"name":"Department of Statistics and Electrical Engineering, Stanford University","place":["Stanford"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"John C.","family":"Duchi","sequence":"additional","affiliation":[{"name":"Department of Statistics and Electrical Engineering, Stanford University","place":["Stanford"]}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"301","published-online":{"date-parts":[[2024,2]]},"reference":[{"key":"e_1_3_3_2_1","doi-asserted-by":"publisher","DOI":"10.1093\/imaiai\/iaaa017"},{"key":"e_1_3_3_3_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10107-017-1125-8"},{"key":"e_1_3_3_4_1","doi-asserted-by":"publisher","DOI":"10.1017\/jpr.2019.49"},{"key":"e_1_3_3_5_1","doi-asserted-by":"publisher","DOI":"10.1287\/moor.2018.0936"},{"key":"e_1_3_3_6_1","doi-asserted-by":"publisher","DOI":"10.1111\/ectj.12097"},{"key":"e_1_3_3_7_1","doi-asserted-by":"publisher","DOI":"10.1920\/wp.cem.2018.1618"},{"key":"e_1_3_3_8_1","first-page":"299","article-title":"\u201cInformation-Type Measures of Difference of Probability Distributions and Indirect Observation,\u201d","volume":"2","author":"Csisz\u00e1r I.","year":"1967","unstructured":"Csisz\u00e1r, I. (1967), \u201cInformation-Type Measures of Difference of Probability Distributions and Indirect Observation,\u201d Studia Scientifica Mathematica Hungary, 2, 299\u2013318.","journal-title":"Studia Scientifica Mathematica Hungary"},{"key":"e_1_3_3_9_1","unstructured":"Dua D. and Graff C. (2017) \u201cUCI Machine Learning Repository.\u201d"},{"key":"e_1_3_3_10_1","doi-asserted-by":"publisher","DOI":"10.1214\/20-AOS2004"},{"key":"e_1_3_3_11_1","unstructured":"Gupta S. and Rothenh\u00e4usler D. (2021) \u201cThe s-value: Evaluating Stability with Respect to Distributional Shifts \u201d arXiv:2105.03067 [stat.ME]."},{"key":"e_1_3_3_12_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-387-84858-7"},{"key":"e_1_3_3_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_3_14_1","doi-asserted-by":"publisher","DOI":"10.1111\/biom.12101"},{"key":"e_1_3_3_15_1","doi-asserted-by":"publisher","DOI":"10.1257\/000282803321946921"},{"key":"e_1_3_3_16_1","unstructured":"Jeong S. and Namkoong H. (2020) \u201cRobust Causal Inference Under Covariate Shift via Worst-Case Subpopulation Treatment Effects \u201d arXiv:2007.02411 [stat.ML]."},{"key":"e_1_3_3_17_1","doi-asserted-by":"publisher","DOI":"10.1162\/99608f92.f06c6e61"},{"key":"e_1_3_3_18_1","volume-title":"Technical Report","author":"Krizhevsky A.","year":"2009","unstructured":"Krizhevsky, A., and Hinton, G. (2009), \u201cLearning Multiple Layers of Features from Tiny Images,\u201d Technical Report, University of Toronto."},{"key":"e_1_3_3_19_1","unstructured":"LeCun Y. Cortes C. and Burges C. (1998) \u201cMNIST Handwritten Digit Database \u201d ATT Labs [Online]."},{"key":"e_1_3_3_20_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.2017.1307116"},{"key":"e_1_3_3_21_1","volume-title":"Proceedings of the 11th Annual European Symposium on Algorithms","author":"min Chung K.","year":"2003","unstructured":"min Chung, K., and Lu, H.-I. (2003), \u201cAn Optimal Algorithm for the Maximum-Density Segment Problem,\u201d in Proceedings of the 11th Annual European Symposium on Algorithms."},{"key":"e_1_3_3_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.2010.2068870"},{"key":"e_1_3_3_23_1","volume-title":"Dataset Shift in Machine Learning","author":"Quionero-Candela J.","year":"2009","unstructured":"Quionero-Candela, J., Sugiyama, M., Schwaighofer, A., and Lawrence, N. D. (2009), Dataset Shift in Machine Learning, Cambridge, MA: The MIT Press."},{"key":"e_1_3_3_24_1","volume-title":"Proceedings of the 36th International Conference on Machine Learning","author":"Recht B.","year":"2019","unstructured":"Recht, B., Roelofs, R., Schmidt, L., and Shankar, V. (2019), \u201cDo ImageNet Classifiers Generalize to ImageNet?\u201d in Proceedings of the 36th International Conference on Machine Learning."},{"key":"e_1_3_3_25_1","doi-asserted-by":"publisher","DOI":"10.21314\/JOR.2000.038"},{"key":"e_1_3_3_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"key":"e_1_3_3_27_1","volume-title":"Proceedings of the Eighth International Conference on Learning Representations","author":"Sagawa S.","year":"2020","unstructured":"Sagawa, S., Koh, P. W., Hashimoto, T. B., and Liang, P. (2020), \u201cDistributionally Robust Neural Networks for Group Shifts: On the Importance of Regularization for Worst-Case Generalization,\u201d in Proceedings of the Eighth International Conference on Learning Representations."},{"key":"e_1_3_3_28_1","volume-title":"Proceedings of the 24th International Conference on Artificial Intelligence and Statistics","author":"Subbaswamy A.","year":"2021","unstructured":"Subbaswamy, A., Adams, R., and Saria, S. (2021), \u201cEvaluating Model Robustness to Dataset Shift,\u201d in Proceedings of the 24th International Conference on Artificial Intelligence and Statistics."},{"key":"e_1_3_3_29_1","first-page":"985","article-title":"\u201cCovariate Shift Adaptation by Importance Weighted Cross Validation,\u201d","volume":"8","author":"Sugiyama M.","year":"2007","unstructured":"Sugiyama, M., Krauledat, M., and M\u00fcller, K.-R. (2007), \u201cCovariate Shift Adaptation by Importance Weighted Cross Validation,\u201d Journal of Machine Learning Research, 8, 985\u20131005.","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_3_30_1","article-title":"\u201cWhen Robustness Doesn\u2019t Promote Robustness: Synthetic vs","author":"Taori R.","year":"2020","unstructured":"Taori, R., Dave, A., Shankar, V., Carlini, N., Recht, B., and Schmidt, L. (2020), \u201cWhen Robustness Doesn\u2019t Promote Robustness: Synthetic vs. Natural Distribution Shifts on ImageNet,\u201d under review.","journal-title":"Natural Distribution Shifts on ImageNet,\u201d under review"},{"key":"e_1_3_3_31_1","article-title":"\u201cConformal Prediction Under Covariate Shift,\u201d","volume":"32","author":"Tibshirani R. J.","year":"2019","unstructured":"Tibshirani, R. J., Barber, R. F., Cand\u00e8s, E. J., and Ramdas, A. (2019), \u201cConformal Prediction Under Covariate Shift,\u201d in Advances in Neural Information Processing Systems (Vol. 32).","journal-title":"in Advances in Neural Information Processing Systems"},{"key":"e_1_3_3_32_1","first-page":"10999","article-title":"\u201cSense and Sensitivity Analysis: Simple Post-hoc Analysis of Bias Due to Unobserved Confounding,\u201d in","volume":"33","author":"Veitch V.","year":"2020","unstructured":"Veitch, V., and Zaveri, A. (2020), \u201cSense and Sensitivity Analysis: Simple Post-hoc Analysis of Bias Due to Unobserved Confounding,\u201d in Advances in Neural Information Processing Systems (Vol. 33), pp. 10999\u201311009.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_3_33_1","volume-title":"Algorithmic Learning in a Random World","author":"Vovk V.","year":"2005","unstructured":"Vovk, V., Grammerman, A., and Shafer, G. (2005), Algorithmic Learning in a Random World, New York: Springer."},{"key":"e_1_3_3_34_1","article-title":"\u201cCold Case: The Lost MNIST Digits,\u201d","volume":"32","author":"Yadav C.","year":"2019","unstructured":"Yadav, C., and Bottou, L. (2019), \u201cCold Case: The Lost MNIST Digits,\u201d in Advances in Neural Information Processing Systems (Vol. 32).","journal-title":"in Advances in Neural Information Processing Systems"}],"container-title":["Journal of the American Statistical Association"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.tandfonline.com\/doi\/pdf\/10.1080\/01621459.2023.2298037","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,2,28]],"date-time":"2025-02-28T18:21:34Z","timestamp":1740766894000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.tandfonline.com\/doi\/full\/10.1080\/01621459.2023.2298037"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,2]]},"references-count":33,"journal-issue":{"issue":"548","published-print":{"date-parts":[[2024,10]]}},"alternative-id":["10.1080\/01621459.2023.2298037"],"URL":"https:\/\/doi.org\/10.1080\/01621459.2023.2298037","relation":{},"ISSN":["0162-1459","1537-274X"],"issn-type":[{"value":"0162-1459","type":"print"},{"value":"1537-274X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,2]]},"assertion":[{"value":"The publishing and review policy for this title is described in its Aims & Scope.","order":1,"name":"peerreview_statement","label":"Peer Review Statement"},{"value":"http:\/\/www.tandfonline.com\/action\/journalInformation?show=aimsScope&journalCode=uasa20","URL":"http:\/\/www.tandfonline.com\/action\/journalInformation?show=aimsScope&journalCode=uasa20","order":2,"name":"aims_and_scope_url","label":"Aim & Scope"},{"value":"2023-03-06","order":0,"name":"received","label":"Received","group":{"name":"publication_history","label":"Publication History"}},{"value":"2023-10-10","order":1,"name":"revised","label":"Revised","group":{"name":"publication_history","label":"Publication History"}},{"value":"2023-12-17","order":2,"name":"accepted","label":"Accepted","group":{"name":"publication_history","label":"Publication History"}},{"value":"2024-02-01","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}