{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,15]],"date-time":"2025-10-15T18:16:41Z","timestamp":1760552201722,"version":"3.40.3"},"publisher-location":"Cham","reference-count":106,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031732225"},{"type":"electronic","value":"9783031732232"}],"license":[{"start":{"date-parts":[[2024,11,8]],"date-time":"2024-11-08T00:00:00Z","timestamp":1731024000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,11,8]],"date-time":"2024-11-08T00:00:00Z","timestamp":1731024000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-3-031-73223-2_9","type":"book-chapter","created":{"date-parts":[[2024,11,7]],"date-time":"2024-11-07T18:47:45Z","timestamp":1731005265000},"page":"140-158","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["A Framework for\u00a0Efficient Model Evaluation Through Stratification, Sampling, and\u00a0Estimation"],"prefix":"10.1007","author":[{"given":"Riccardo","family":"Fogliato","sequence":"first","affiliation":[]},{"given":"Pratik","family":"Patil","sequence":"additional","affiliation":[]},{"given":"Mathew","family":"Monfort","sequence":"additional","affiliation":[]},{"given":"Pietro","family":"Perona","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,11,8]]},"reference":[{"issue":"6671","key":"9_CR1","doi-asserted-by":"publisher","first-page":"669","DOI":"10.1126\/science.adi6000","volume":"382","author":"AN Angelopoulos","year":"2023","unstructured":"Angelopoulos, A.N., Bates, S., Fannjiang, C., Jordan, M.I., Zrnic, T.: Prediction-powered inference. Science 382(6671), 669\u2013674 (2023)","journal-title":"Science"},{"key":"9_CR2","doi-asserted-by":"crossref","unstructured":"Angelopoulos, A.N., Duchi, J.C., Zrnic, T.: Ppi++: Efficient prediction-powered inference. arXiv preprint arXiv:2311.01453 (2023)","DOI":"10.1126\/science.adi6000"},{"key":"9_CR3","unstructured":"Ash, J.T., Zhang, C., Krishnamurthy, A., Langford, J., Agarwal, A.: Deep batch active learning by diverse, uncertain gradient lower bounds. arXiv preprint arXiv:1906.03671 (2019)"},{"key":"9_CR4","first-page":"19274","volume":"35","author":"C Baek","year":"2022","unstructured":"Baek, C., Jiang, Y., Raghunathan, A., Kolter, J.Z.: Agreement-on-the-line: predicting the performance of neural networks under distribution shift. Adv. Neural. Inf. Process. Syst. 35, 19274\u201319289 (2022)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"9_CR5","unstructured":"Barbu, A., et al.: Objectnet: a large-scale bias-controlled dataset for pushing the limits of object recognition models. Adv. Neural Inform. Process. Syst. 32 (2019)"},{"key":"9_CR6","unstructured":"Beery, S., Cole, E., Gjoka, A.: The iwildcam 2020 competition dataset. arXiv preprint arXiv:2004.10340 (2020)"},{"issue":"4","key":"9_CR7","doi-asserted-by":"publisher","first-page":"831","DOI":"10.1093\/biomet\/92.4.831","volume":"92","author":"FJ Breidt","year":"2005","unstructured":"Breidt, F.J., Claeskens, G., Opsomer, J.: Model-assisted estimation for complex surveys using penalised splines. Biometrika 92(4), 831\u2013846 (2005)","journal-title":"Biometrika"},{"issue":"2","key":"9_CR8","doi-asserted-by":"publisher","first-page":"190","DOI":"10.1214\/16-STS589","volume":"32","author":"FJ Breidt","year":"2017","unstructured":"Breidt, F.J., Opsomer, J.D.: Model-assisted survey estimation with modern prediction techniques. Stat. Sci. 32(2), 190\u2013205 (2017). https:\/\/doi.org\/10.1214\/16-STS589","journal-title":"Stat. Sci."},{"key":"9_CR9","doi-asserted-by":"crossref","unstructured":"Brus, D.J.: Spatial sampling with R. CRC Press (2022)","DOI":"10.1201\/9781003258940"},{"key":"9_CR10","unstructured":"Chen, M., Goel, K., Sohoni, N.S., Poms, F., Fatahalian, K., R\u00e9, C.: Mandoline: Model evaluation under distribution shift. In: International Conference on Machine Learning, pp. 1617\u20131629. PMLR (2021)"},{"issue":"30","key":"9_CR11","doi-asserted-by":"publisher","first-page":"4912","DOI":"10.1002\/sim.8760","volume":"39","author":"T Chen","year":"2020","unstructured":"Chen, T., Lumley, T.: Optimal multiwave sampling for regression modeling in two-phase designs. Stat. Med. 39(30), 4912\u20134921 (2020)","journal-title":"Stat. Med."},{"issue":"8","key":"9_CR12","doi-asserted-by":"publisher","first-page":"1482","DOI":"10.1002\/sim.9300","volume":"41","author":"T Chen","year":"2022","unstructured":"Chen, T., Lumley, T.: Optimal sampling for design-based estimators of regression models. Stat. Med. 41(8), 1482\u20131497 (2022)","journal-title":"Stat. Med."},{"key":"9_CR13","doi-asserted-by":"crossref","unstructured":"Chen, Y., Zhang, S., Song, R.: Scoring your prediction on unseen data. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR) Workshops, pp. 3279\u20133288 (June 2023)","DOI":"10.1109\/CVPRW59228.2023.00330"},{"issue":"10","key":"9_CR14","doi-asserted-by":"publisher","first-page":"1865","DOI":"10.1109\/JPROC.2017.2675998","volume":"105","author":"G Cheng","year":"2017","unstructured":"Cheng, G., Han, J., Lu, X.: Remote sensing image scene classification: benchmark and state of the art. Proc. IEEE 105(10), 1865\u20131883 (2017)","journal-title":"Proc. IEEE"},{"key":"9_CR15","doi-asserted-by":"publisher","unstructured":"Chouldechova, A., Deng, S., Wang, Y., Xia, W., Perona, P.: Unsupervised and semi-supervised bias benchmarking in face recognition. In: European Conference on Computer Vision, pp. 289\u2013306. Springer (2022). https:\/\/doi.org\/10.1007\/978-3-031-19778-9_17","DOI":"10.1007\/978-3-031-19778-9_17"},{"key":"9_CR16","doi-asserted-by":"crossref","unstructured":"Chu, W., Zinkevich, M., Li, L., Thomas, A., Tseng, B.: Unbiased online active learning in data streams. In: Proceedings of the 17th ACM SIGKDD International Conference on Knowledge Discovery And Data Mining, pp. 195\u2013203 (2011)","DOI":"10.1145\/2020408.2020444"},{"key":"9_CR17","unstructured":"Chuang, C.Y., Torralba, A., Jegelka, S.: Estimating generalization under distribution shifts via domain-invariant representations. arXiv preprint arXiv:2007.03511 (2020)"},{"key":"9_CR18","doi-asserted-by":"crossref","unstructured":"Cimpoi, M., Maji, S., Kokkinos, I., Mohamed, S., , Vedaldi, A.: Describing textures in the wild. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR) (2014)","DOI":"10.1109\/CVPR.2014.461"},{"issue":"4","key":"9_CR19","doi-asserted-by":"publisher","first-page":"1733","DOI":"10.1111\/rssa.12916","volume":"185","author":"RG Clark","year":"2022","unstructured":"Clark, R.G., Steel, D.G.: Sample design for analysis using high-influence probability sampling. J. R. Stat. Soc. Ser. A Stat. Soc. 185(4), 1733\u20131756 (2022)","journal-title":"J. R. Stat. Soc. Ser. A Stat. Soc."},{"key":"9_CR20","unstructured":"Coates, A., Ng, A., Lee, H.: An analysis of single-layer networks in unsupervised feature learning. In: Proceedings of the Fourteenth International Conference on Artificial Intelligence and Statistics, pp. 215\u2013223. JMLR Workshop and Conference Proceedings (2011)"},{"key":"9_CR21","unstructured":"Cochran, W.G.: Sampling Techniques. John Wiley & Sons (1977)"},{"key":"9_CR22","doi-asserted-by":"publisher","first-page":"129","DOI":"10.1613\/jair.295","volume":"4","author":"DA Cohn","year":"1996","unstructured":"Cohn, D.A., Ghahramani, Z., Jordan, M.I.: Active learning with statistical models. J. Artif. Intell. Res. 4, 129\u2013145 (1996)","journal-title":"J. Artif. Intell. Res."},{"issue":"6","key":"9_CR23","doi-asserted-by":"publisher","first-page":"141","DOI":"10.1109\/MSP.2012.2211477","volume":"29","author":"L Deng","year":"2012","unstructured":"Deng, L.: The mnist database of handwritten digit images for machine learning research. IEEE Signal Process. Mag. 29(6), 141\u2013142 (2012)","journal-title":"IEEE Signal Process. Mag."},{"key":"9_CR24","unstructured":"Deng, W., Gould, S., Zheng, L.: What does rotation prediction tell us about classifier accuracy under varying testing environments? In: International Conference on Machine Learning, pp. 2579\u20132589. PMLR (2021)"},{"key":"9_CR25","doi-asserted-by":"crossref","unstructured":"Deng, W., Zheng, L.: Are labels always necessary for classifier accuracy evaluation? In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 15069\u201315078 (2021)","DOI":"10.1109\/CVPR46437.2021.01482"},{"key":"9_CR26","unstructured":"Emma, D., Jared, J., Cukierski, W.: Diabetic retinopathy detection (2015). https:\/\/kaggle.com\/competitions\/diabetic-retinopathy-detection"},{"key":"9_CR27","unstructured":"Everingham, M., Van\u00a0Gool, L., Williams, C.K.I., Winn, J., Zisserman, A.: The PASCAL Visual Object Classes Challenge 2007 (VOC 2007) Results. http:\/\/www.pascal-network.org\/challenges\/VOC\/voc2007\/workshop\/index.html"},{"key":"9_CR28","unstructured":"Farquhar, S., Gal, Y., Rainforth, T.: On statistical bias in active learning: How and when to fix it. arXiv preprint arXiv:2101.11665 (2021)"},{"key":"9_CR29","unstructured":"Fei-Fei, L., Fergus, R., Perona, P.: Learning generative visual models from few training examples: An incremental bayesian approach tested on 101 object categories. In: 2004 Conference on Computer Vision and Pattern Recognition Workshop, pp. 178\u2013178. IEEE (2004)"},{"key":"9_CR30","unstructured":"Fuller, W.A.: Sampling Statistics. John Wiley & Sons (2011)"},{"key":"9_CR31","unstructured":"Gal, Y., Islam, R., Ghahramani, Z.: Deep bayesian active learning with image data. In: International Conference on Machine Learning, pp. 1183\u20131192. PMLR (2017)"},{"key":"9_CR32","unstructured":"Ganti, R., Gray, A.: Upal: Unbiased pool based active learning. In: Artificial Intelligence and Statistics, pp. 422\u2013431. PMLR (2012)"},{"key":"9_CR33","unstructured":"Garg, S., Balakrishnan, S., Lipton, Z.C., Neyshabur, B., Sedghi, H.: Leveraging unlabeled data to predict out-of-distribution performance. arXiv preprint arXiv:2201.04234 (2022)"},{"key":"9_CR34","doi-asserted-by":"crossref","unstructured":"Geiger, A., Lenz, P., Stiller, C., Urtasun, R.: Vision meets robotics: the kitti dataset. Inter. J. Robot. Res. (IJRR) (2013)","DOI":"10.1177\/0278364913491297"},{"issue":"1","key":"9_CR35","doi-asserted-by":"publisher","first-page":"73","DOI":"10.1214\/ss\/1023798999","volume":"17","author":"BI Graubardand","year":"2002","unstructured":"Graubardand, B.I., Korn, E.L.: Inference for superpopulation parameters using sample surveys. Stat. Sci. 17(1), 73\u201396 (2002)","journal-title":"Stat. Sci."},{"key":"9_CR36","doi-asserted-by":"crossref","unstructured":"Guillory, D., Shankar, V., Ebrahimi, S., Darrell, T., Schmidt, L.: Predicting with confidence on unseen distributions. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 1134\u20131144 (2021)","DOI":"10.1109\/ICCV48922.2021.00117"},{"issue":"4","key":"9_CR37","doi-asserted-by":"publisher","first-page":"387","DOI":"10.21136\/CPM.1959.117317","volume":"84","author":"J H\u00e1jek","year":"1959","unstructured":"H\u00e1jek, J.: Optimal strategy and other problems in probability sampling. \u010casopis pro p\u011bstov\u00e1n\u00ed matematiky 84(4), 387\u2013423 (1959)","journal-title":"\u010casopis pro p\u011bstov\u00e1n\u00ed matematiky"},{"key":"9_CR38","doi-asserted-by":"crossref","unstructured":"He, K., Zhang, X., Ren, S., Sun, J.: Deep residual learning for image recognition. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 770\u2013778 (2016)","DOI":"10.1109\/CVPR.2016.90"},{"key":"9_CR39","unstructured":"H\u00e9bert-Johnson, U., Kim, M., Reingold, O., Rothblum, G.: Multicalibration: calibration for the (computationally-identifiable) masses. In: International Conference on Machine Learning, pp. 1939\u20131948. PMLR (2018)"},{"key":"9_CR40","doi-asserted-by":"crossref","unstructured":"Helber, P., Bischke, B., Dengel, A., Borth, D.: Eurosat: a novel dataset and deep learning benchmark for land use and land cover classification. IEEE J. Selected Topics Appli. Earth Observations Remote Sensing (2019)","DOI":"10.1109\/JSTARS.2019.2918242"},{"key":"9_CR41","doi-asserted-by":"crossref","unstructured":"Hendrycks, D., et al.: The many faces of robustness: a critical analysis of out-of-distribution generalization. In: ICCV (2021)","DOI":"10.1109\/ICCV48922.2021.00823"},{"key":"9_CR42","doi-asserted-by":"crossref","unstructured":"Hendrycks, D., Zhao, K., Basart, S., Steinhardt, J., Song, D.: Natural adversarial examples. In: CVPR (2021)","DOI":"10.1109\/CVPR46437.2021.01501"},{"issue":"260","key":"9_CR43","doi-asserted-by":"publisher","first-page":"663","DOI":"10.1080\/01621459.1952.10483446","volume":"47","author":"DG Horvitz","year":"1952","unstructured":"Horvitz, D.G., Thompson, D.J.: A generalization of sampling without replacement from a finite universe. J. Am. Stat. Assoc. 47(260), 663\u2013685 (1952)","journal-title":"J. Am. Stat. Assoc."},{"key":"9_CR44","doi-asserted-by":"publisher","unstructured":"Ilharco, G., et al.: Openclip (2021). https:\/\/doi.org\/10.5281\/zenodo.5143773","DOI":"10.5281\/zenodo.5143773"},{"key":"9_CR45","unstructured":"Imberg, H., Axelson-Fisk, M., Jonasson, J.: Optimal subsampling designs. arXiv preprint arXiv:2304.03019 (2023)"},{"key":"9_CR46","unstructured":"Imberg, H., Jonasson, J., Axelson-Fisk, M.: Optimal sampling in unbiased active learning. In: International Conference on Artificial Intelligence and Statistics, pp. 559\u2013569. PMLR (2020)"},{"key":"9_CR47","unstructured":"Imberg, H., Yang, X., Flannagan, C., B\u00e4rgman, J.: Active sampling: A machine-learning-assisted framework for finite population inference with optimal subsamples. arXiv preprint arXiv:2212.10024 (2022)"},{"issue":"377","key":"9_CR48","doi-asserted-by":"publisher","first-page":"89","DOI":"10.1080\/01621459.1982.10477770","volume":"77","author":"CT Isaki","year":"1982","unstructured":"Isaki, C.T., Fuller, W.A.: Survey design under the regression superpopulation model. J. Am. Stat. Assoc. 77(377), 89\u201396 (1982)","journal-title":"J. Am. Stat. Assoc."},{"key":"9_CR49","unstructured":"Jiang, Y., Nagarajan, V., Baek, C., Kolter, J.Z.: Assessing generalization of sgd via disagreement. arXiv preprint arXiv:2106.13799 (2021)"},{"key":"9_CR50","doi-asserted-by":"crossref","unstructured":"Johnson, J., Hariharan, B., Van Der\u00a0Maaten, L., Fei-Fei, L., Lawrence\u00a0Zitnick, C., Girshick, R.: Clevr: a diagnostic dataset for compositional language and elementary visual reasoning. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 2901\u20132910 (2017)","DOI":"10.1109\/CVPR.2017.215"},{"issue":"4","key":"9_CR51","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2108097119","volume":"119","author":"MP Kim","year":"2022","unstructured":"Kim, M.P., Kern, C., Goldwasser, S., Kreuter, F., Reingold, O.: Universal adaptability: target-independent inference that competes with propensity scoring. Proc. Nat. Acad. Sci. 119(4), e2108097119 (2022)","journal-title":"Proc. Nat. Acad. Sci."},{"key":"9_CR52","unstructured":"Kirsch, A., Van\u00a0Amersfoort, J., Gal, Y.: Batchbald: efficient and diverse batch acquisition for deep bayesian active learning. Adv. Neural Inform. Process. Syst. 32 (2019)"},{"key":"9_CR53","unstructured":"Koh, P.W., et\u00a0al.: Wilds: a benchmark of in-the-wild distribution shifts. In: International Conference on Machine Learning, pp. 5637\u20135664. PMLR (2021)"},{"key":"9_CR54","unstructured":"Kossen, J., Farquhar, S., Gal, Y., Rainforth, T.: Active surrogate estimators: an active learning approach to label-efficient model evaluation. In: Koyejo, S., Mohamed, S., Agarwal, A., Belgrave, D., Cho, K., Oh, A. (eds.) Advances in Neural Information Processing Systems, vol.\u00a035, pp. 24557\u201324570. Curran Associates, Inc. (2022)"},{"key":"9_CR55","unstructured":"Kossen, J., Farquhar, S., Gal, Y., Rainforth, T.: Active testing: sample-efficient model evaluation. In: International Conference on Machine Learning, pp. 5753\u20135763. PMLR (2021)"},{"key":"9_CR56","doi-asserted-by":"crossref","unstructured":"Krause, J., Stark, M., Deng, J., Fei-Fei, L.: 3d object representations for fine-grained categorization. In: 4th International IEEE Workshop on 3D Representation and Recognition (3dRR-13), Sydney, Australia (2013)","DOI":"10.1109\/ICCVW.2013.77"},{"key":"9_CR57","unstructured":"Krizhevsky, A., Hinton, G., et\u00a0al.: Learning multiple layers of features from tiny images (2009)"},{"key":"9_CR58","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"68","DOI":"10.1007\/978-3-319-23528-8_5","volume-title":"Machine Learning and Knowledge Discovery in Databases","author":"M Kull","year":"2015","unstructured":"Kull, M., Flach, P.: Novel decompositions of proper scoring rules for classification: score adjustment as precursor to calibration. In: Appice, A., Rodrigues, P.P., Santos Costa, V., Soares, C., Gama, J., Jorge, A. (eds.) ECML PKDD 2015. LNCS (LNAI), vol. 9284, pp. 68\u201385. Springer, Cham (2015). https:\/\/doi.org\/10.1007\/978-3-319-23528-8_5"},{"key":"9_CR59","unstructured":"LAION AI: Clip benchmark. https:\/\/github.com\/LAION-AI\/CLIP_benchmark"},{"key":"9_CR60","doi-asserted-by":"crossref","unstructured":"LeCun, Y., Huang, F.J., Bottou, L.: Learning methods for generic object recognition with invariance to pose and lighting. In: Proceedings of the 2004 IEEE Computer Society Conference on Computer Vision and Pattern Recognition, CVPR 2004. vol.\u00a02, pp. II\u2013104. IEEE (2004)","DOI":"10.1109\/CVPR.2004.1315150"},{"key":"9_CR61","doi-asserted-by":"crossref","unstructured":"Lewis, D.D.: A sequential algorithm for training text classifiers: corrigendum and additional data. In: ACM SIGIR Forum, vol.\u00a029, pp. 13\u201319. ACM New York (1995)","DOI":"10.1145\/219587.219592"},{"key":"9_CR62","doi-asserted-by":"crossref","unstructured":"Lewis, D.D., Catlett, J.: Heterogeneous uncertainty sampling for supervised learning. In: Machine Learning Proceedings 1994, pp. 148\u2013156. Elsevier (1994)","DOI":"10.1016\/B978-1-55860-335-6.50026-X"},{"key":"9_CR63","unstructured":"Li, Z., Ma, X., Xu, C., Cao, C., Xu, J., L\u00fc, J.: Boosting operational dnn testing efficiency through conditioning 10(1145\/3338906), 3338930 (2019)"},{"key":"9_CR64","doi-asserted-by":"crossref","unstructured":"Liu, Z., Mao, H., Wu, C.Y., Feichtenhofer, C., Darrell, T., Xie, S.: A convnet for the 2020s. In: Proceedings of the IEEE\/CVF Conference On Computer Vision and Pattern Recognition, pp. 11976\u201311986 (2022)","DOI":"10.1109\/CVPR52688.2022.01167"},{"key":"9_CR65","doi-asserted-by":"crossref","unstructured":"Lohr, S.L.: Sampling: design and analysis. CRC press (2021)","DOI":"10.1201\/9780429298899"},{"issue":"2","key":"9_CR66","doi-asserted-by":"publisher","first-page":"200","DOI":"10.1111\/j.1751-5823.2011.00138.x","volume":"79","author":"T Lumley","year":"2011","unstructured":"Lumley, T., Shaw, P.A., Dai, J.Y.: Connections between survey calibration estimators and semiparametric models for incomplete data. Int. Stat. Rev. 79(2), 200\u2013220 (2011)","journal-title":"Int. Stat. Rev."},{"key":"9_CR67","unstructured":"Maji, S., Rahtu, E., Kannala, J., Blaschko, M., Vedaldi, A.: Fine-grained visual classification of aircraft. arXiv preprint arXiv:1306.5151 (2013)"},{"key":"9_CR68","unstructured":"Matthey, L., Higgins, I., Hassabis, D., Lerchner, A.: dsprites: Disentanglement testing sprites dataset (2017). https:\/\/github.com\/deepmind\/dsprites-dataset\/"},{"issue":"2","key":"9_CR69","doi-asserted-by":"publisher","first-page":"131","DOI":"10.1093\/jssam\/smw041","volume":"5","author":"KS McConville","year":"2017","unstructured":"McConville, K.S., Breidt, F.J., Lee, T.C., Moisen, G.G.: Model-assisted survey regression estimation with the lasso. J. Surv. Statist. Methodol. 5(2), 131\u2013158 (2017)","journal-title":"J. Surv. Statist. Methodol."},{"key":"9_CR70","unstructured":"Miller, B.A., Vila, J., Kirn, M., Zipkin, J.R.: Classifier performance estimation with unbalanced, partially labeled data. In: Torgo, L., Matwin, S., Weiss, G., Moniz, N., Branco, P. (eds.) Proceedings of The International Workshop on Cost-Sensitive Learning. Proceedings of Machine Learning Research, 05 May, vol.\u00a088, pp. 4\u201316. PMLR (2018)"},{"key":"9_CR71","unstructured":"Miller, J.P., et al.: Accuracy on the line: on the strong correlation between out-of-distribution and in-distribution generalization. In: International Conference on Machine Learning, pp. 7721\u20137735. PMLR (2021)"},{"key":"9_CR72","unstructured":"Netzer, Y., Wang, T., Coates, A., Bissacco, A., Wu, B., Ng, A.Y.: Reading digits in natural images with unsupervised feature learning (2011)"},{"key":"9_CR73","doi-asserted-by":"publisher","unstructured":"Neyman, J.: On the two different aspects of the representative method: the method of stratified sampling and the method of purposive selection. In: Breakthroughs in Statistics: Methodology and Distribution, pp. 123\u2013150. Springer (1992). https:\/\/doi.org\/10.1007\/978-1-4612-4380-9_12","DOI":"10.1007\/978-1-4612-4380-9_12"},{"key":"9_CR74","doi-asserted-by":"crossref","unstructured":"Nilsback, M.E., Zisserman, A.: Automated flower classification over a large number of classes. In: 2008 Sixth Indian Conference on Computer Vision, Graphics & Image Processing, pp. 722\u2013729. IEEE (2008)","DOI":"10.1109\/ICVGIP.2008.47"},{"key":"9_CR75","doi-asserted-by":"crossref","unstructured":"Parkhi, O.M., Vedaldi, A., Zisserman, A., Jawahar, C.: Cats and dogs. In: 2012 IEEE conference on Computer Vision and Pattern Recognition, pp. 3498\u20133505. IEEE (2012)","DOI":"10.1109\/CVPR.2012.6248092"},{"key":"9_CR76","doi-asserted-by":"crossref","unstructured":"Poms, F., et al.: Low-shot validation: active importance sampling for estimating classifier performance on rare categories. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision (ICCV), pp. 10705\u201310714 (October 2021)","DOI":"10.1109\/ICCV48922.2021.01053"},{"key":"9_CR77","unstructured":"Radford, A., et\u00a0al.: Learning transferable visual models from natural language supervision. In: International Conference on Machine Learning, pp. 8748\u20138763. PMLR (2021)"},{"key":"9_CR78","unstructured":"Recht, B., Roelofs, R., Schmidt, L., Shankar, V.: Do imagenet classifiers generalize to imagenet? In: International Conference on Machine Learning, pp. 5389\u20135400. PMLR (2019)"},{"issue":"9","key":"9_CR79","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3472291","volume":"54","author":"P Ren","year":"2021","unstructured":"Ren, P., et al.: A survey of deep active learning. ACM Comput. Surv. (CSUR) 54(9), 1\u201340 (2021)","journal-title":"ACM Comput. Surv. (CSUR)"},{"key":"9_CR80","unstructured":"Roth, A.: Uncertain: Modern topics in uncertainty estimation (2022)"},{"issue":"3","key":"9_CR81","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1007\/s11263-015-0816-y","volume":"115","author":"O Russakovsky","year":"2015","unstructured":"Russakovsky, O., et al.: ImageNet Large Scale Visual Recognition Challenge. Inter. J. Comput. Vis. (IJCV) 115(3), 211\u2013252 (2015). https:\/\/doi.org\/10.1007\/s11263-015-0816-y","journal-title":"Inter. J. Comput. Vis. (IJCV)"},{"issue":"2","key":"9_CR82","first-page":"99","volume":"33","author":"CE S\u00e4rndal","year":"2007","unstructured":"S\u00e4rndal, C.E.: The calibration approach in survey theory and practice. Surv. Pract. 33(2), 99\u2013119 (2007)","journal-title":"Surv. Pract."},{"key":"9_CR83","unstructured":"S\u00e4rndal, C.E., Swensson, B., Wretman, J.: Model assisted survey sampling. Springer Science & Business Media (2003)"},{"key":"9_CR84","unstructured":"Sawade, C., Landwehr, N., Bickel, S., Scheffer, T.: Active risk estimation. In: Proceedings of the 27th International Conference on International Conference on Machine Learning, ICML 2010. pp. 951-958. Omnipress, Madison, WI, USA (2010)"},{"key":"9_CR85","unstructured":"Sawade, C., Landwehr, N., Scheffer, T.: Active estimation of f-measures. In: Lafferty, J., Williams, C., Shawe-Taylor, J., Zemel, R., Culotta, A. (eds.) Advances in Neural Information Processing Systems, vol.\u00a023. Curran Associates, Inc. (2010)"},{"key":"9_CR86","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"309","DOI":"10.1007\/3-540-44816-0_31","volume-title":"Advances in Intelligent Data Analysis","author":"T Scheffer","year":"2001","unstructured":"Scheffer, T., Decomain, C., Wrobel, S.: Active hidden markov models for information extraction. In: Hoffmann, F., Hand, D.J., Adams, N., Fisher, D., Guimaraes, G. (eds.) IDA 2001. LNCS, vol. 2189, pp. 309\u2013318. Springer, Heidelberg (2001). https:\/\/doi.org\/10.1007\/3-540-44816-0_31"},{"key":"9_CR87","unstructured":"Schuhmann, C., et al.: LAION-5b: an open large-scale dataset for training next generation image-text models. In: Thirty-sixth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (2022). https:\/\/openreview.net\/forum?id=M3Y74vmsMcY"},{"key":"9_CR88","unstructured":"Sener, O., Savarese, S.: Active learning for convolutional neural networks: A core-set approach. arXiv preprint arXiv:1708.00489 (2017)"},{"key":"9_CR89","unstructured":"Settles, B.: Active learning literature survey (2009)"},{"key":"9_CR90","doi-asserted-by":"crossref","unstructured":"Siddhant, A., Lipton, Z.C.: Deep bayesian active learning for natural language processing: Results of a large-scale empirical study. arXiv preprint arXiv:1808.05697 (2018)","DOI":"10.18653\/v1\/D18-1318"},{"key":"9_CR91","doi-asserted-by":"crossref","unstructured":"Socher, R., et al.: Recursive deep models for semantic compositionality over a sentiment treebank. In: Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing, pp. 1631\u20131642 (2013)","DOI":"10.18653\/v1\/D13-1170"},{"key":"9_CR92","doi-asserted-by":"crossref","unstructured":"Stallkamp, J., Schlipsing, M., Salmen, J., Igel, C.: The german traffic sign recognition benchmark: a multi-class classification competition. In: The 2011 International Joint Conference on Neural Networks, pp. 1453\u20131460. IEEE (2011)","DOI":"10.1109\/IJCNN.2011.6033395"},{"key":"9_CR93","unstructured":"Taylor, J., Earnshaw, B., Mabey, B., Victors, M., Yosinski, J.: Rxrx1: an image set for cellular morphological variation across many experimental batches. In: International Conference on Learning Representations (ICLR) (2019)"},{"key":"9_CR94","doi-asserted-by":"crossref","unstructured":"Till\u00e9, Y.: Sampling and estimation from finite populations. John Wiley & Sons (2020)","DOI":"10.1002\/9781119071259"},{"key":"9_CR95","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"210","DOI":"10.1007\/978-3-030-00934-2_24","volume-title":"Medical Image Computing and Computer Assisted Intervention \u2013 MICCAI 2018","author":"BS Veeling","year":"2018","unstructured":"Veeling, B.S., Linmans, J., Winkens, J., Cohen, T., Welling, M.: Rotation equivariant CNNs for digital pathology. In: Frangi, A.F., Schnabel, J.A., Davatzikos, C., Alberola-L\u00f3pez, C., Fichtinger, G. (eds.) MICCAI 2018. LNCS, vol. 11071, pp. 210\u2013218. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-030-00934-2_24"},{"key":"9_CR96","first-page":"2215","volume":"34","author":"Y Wald","year":"2021","unstructured":"Wald, Y., Feder, A., Greenfeld, D., Shalit, U.: On calibration and out-of-domain generalization. Adv. Neural. Inf. Process. Syst. 34, 2215\u20132227 (2021)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"9_CR97","unstructured":"Wang, H., Ge, S., Lipton, Z., Xing, E.P.: Learning robust global representations by penalizing local predictive power. Adv. Neural Inform. Process. Syst., 10506\u201310518 (2019)"},{"key":"9_CR98","doi-asserted-by":"crossref","unstructured":"Welinder, P., Welling, M., Perona, P.: A lazy man\u2019s approach to benchmarking: Semisupervised classifier evaluation and recalibration. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR) (June 2013)","DOI":"10.1109\/CVPR.2013.419"},{"key":"9_CR99","first-page":"7181","volume":"35","author":"F Wenzel","year":"2022","unstructured":"Wenzel, F., et al.: Assaying out-of-distribution generalization in transfer learning. Adv. Neural. Inf. Process. Syst. 35, 7181\u20137198 (2022)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"issue":"453","key":"9_CR100","doi-asserted-by":"publisher","first-page":"185","DOI":"10.1198\/016214501750333054","volume":"96","author":"C Wu","year":"2001","unstructured":"Wu, C., Sitter, R.R.: A model-calibration approach to using complete auxiliary information from survey data. J. Am. Stat. Assoc. 96(453), 185\u2013193 (2001)","journal-title":"J. Am. Stat. Assoc."},{"key":"9_CR101","doi-asserted-by":"publisher","unstructured":"Xiao, J., Hays, J., Ehinger, K.A., Oliva, A., Torralba, A.: Sun database: large-scale scene recognition from abbey to zoo. In: 2010 IEEE Computer Society Conference on Computer Vision and Pattern Recognition, pp. 3485\u20133492 (June 2010). https:\/\/doi.org\/10.1109\/CVPR.2010.5539970","DOI":"10.1109\/CVPR.2010.5539970"},{"key":"9_CR102","first-page":"27510","volume":"35","author":"Y Yu","year":"2022","unstructured":"Yu, Y., Bates, S., Ma, Y., Jordan, M.: Robust calibration with multi-domain temperature scaling. Adv. Neural. Inf. Process. Syst. 35, 27510\u201327523 (2022)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"9_CR103","unstructured":"Yu, Y., Yang, Z., Wei, A., Ma, Y., Steinhardt, J.: Predicting out-of-distribution error with the projection norm. In: Chaudhuri, K., Jegelka, S., Song, L., Szepesvari, C., Niu, G., Sabato, S. (eds.) Proceedings of the 39th International Conference on Machine Learning, 17\u201323 Jul. Proceedings of Machine Learning Research, vol.\u00a0162, pp. 25721\u201325746. PMLR (2022)"},{"key":"9_CR104","unstructured":"Zhai, Xet al.: The visual task adaptation benchmark (2020). https:\/\/openreview.net\/forum?id=BJena3VtwS"},{"key":"9_CR105","unstructured":"Zrnic, T., Cand\u00e8s, E.J.: Active statistical inference. arXiv preprint arXiv:2403.03208 (2024)"},{"issue":"15","key":"9_CR106","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2322083121","volume":"121","author":"T Zrnic","year":"2024","unstructured":"Zrnic, T., Cand\u00e8s, E.J.: Cross-prediction-powered inference. Proc. Nat. Acad. Sci. 121(15), e2322083121 (2024)","journal-title":"Proc. Nat. Acad. Sci."}],"container-title":["Lecture Notes in Computer Science","Computer Vision \u2013 ECCV 2024"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-73223-2_9","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,30]],"date-time":"2024-11-30T23:22:19Z","timestamp":1733008939000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-73223-2_9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,11,8]]},"ISBN":["9783031732225","9783031732232"],"references-count":106,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-73223-2_9","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2024,11,8]]},"assertion":[{"value":"8 November 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECCV","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Computer Vision","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Milan","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Italy","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"4 October 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"eccv2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/eccv2024.ecva.net\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}