{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,5]],"date-time":"2026-04-05T03:31:32Z","timestamp":1775359892476,"version":"3.50.1"},"reference-count":117,"publisher":"Springer Science and Business Media LLC","issue":"8","license":[{"start":{"date-parts":[[2022,8,17]],"date-time":"2022-08-17T00:00:00Z","timestamp":1660694400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,8,17]],"date-time":"2022-08-17T00:00:00Z","timestamp":1660694400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nat Mach Intell"],"DOI":"10.1038\/s42256-022-00516-1","type":"journal-article","created":{"date-parts":[[2022,8,17]],"date-time":"2022-08-17T16:07:57Z","timestamp":1660752477000},"page":"669-677","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":443,"title":["Advances, challenges and opportunities in creating data for trustworthy AI"],"prefix":"10.1038","volume":"4","author":[{"given":"Weixin","family":"Liang","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2648-9102","authenticated-orcid":false,"given":"Girmaw Abebe","family":"Tadesse","sequence":"additional","affiliation":[]},{"given":"Daniel","family":"Ho","sequence":"additional","affiliation":[]},{"given":"L.","family":"Fei-Fei","sequence":"additional","affiliation":[]},{"given":"Matei","family":"Zaharia","sequence":"additional","affiliation":[]},{"given":"Ce","family":"Zhang","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8880-4764","authenticated-orcid":false,"given":"James","family":"Zou","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2022,8,17]]},"reference":[{"key":"516_CR1","doi-asserted-by":"publisher","first-page":"436","DOI":"10.1038\/nature14539","volume":"521","author":"Y LeCun","year":"2015","unstructured":"LeCun, Y., Bengio, Y. & Hinton, G. Deep learning. Nature 521, 436\u2013444 (2015).","journal-title":"Nature"},{"key":"516_CR2","doi-asserted-by":"publisher","first-page":"252","DOI":"10.1038\/s41586-020-2145-8","volume":"580","author":"D Ouyang","year":"2020","unstructured":"Ouyang, D. et al. Video-based AI for beat-to-beat assessment of cardiac function. Nature 580, 252\u2013256 (2020).","journal-title":"Nature"},{"key":"516_CR3","doi-asserted-by":"publisher","first-page":"22","DOI":"10.1038\/d41586-021-00530-0","volume":"591","author":"M Hutson","year":"2021","unstructured":"Hutson, M. Robo-writers: the rise and risks of language-generating AI. Nature 591, 22\u201325 (2021).","journal-title":"Nature"},{"key":"516_CR4","first-page":"8026","volume":"32","author":"A Paszke","year":"2019","unstructured":"Paszke, A. et al. PyTorch: an imperative style, high-performance deep learning library. Adv. Neural Inf. Process. Syst. 32, 8026\u20138037 (2019).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"516_CR5","unstructured":"Abadi, M. et al. TensorFlow: a system for large-scale machine learning. In Proc. 12th USENIX Symposium on Operating Systems Design and Implementation 265\u2013283 (USENIX Association, 2016)."},{"key":"516_CR6","doi-asserted-by":"crossref","unstructured":"Zhang, X. et al. Dnnbuilder: an automated tool for building high-performance dnn hardware accelerators for fpgas. In 2018 IEEE\/ACM International Conference on Computer-Aided Design (ICCAD) 1\u20138 (IEEE, 2018).","DOI":"10.1145\/3240765.3240801"},{"key":"516_CR7","unstructured":"Code-free machine learning: AutoML with AutoGluon, Amazon SageMaker, and AWS Lambda. AWS Machine Learning Blog https:\/\/aws.amazon.com\/blogs\/machine-learning\/code-free-machine-learning-automl-with-autogluon-amazon-sagemaker-and-aws-lambda\/ (2020)."},{"key":"516_CR8","doi-asserted-by":"publisher","first-page":"288","DOI":"10.1038\/s42256-021-00305-2","volume":"3","author":"E Korot","year":"2021","unstructured":"Korot, E. et al. Code-free deep learning for multi-modality medical image classification. Nat. Mach. Intell. 3, 288\u2013298 (2021).","journal-title":"Nat. Mach. Intell."},{"key":"516_CR9","unstructured":"Dimensional Research. What Data Scientists Tell Us About AI Model Training Today. Alegion https:\/\/content.alegion.com\/dimensional-researchs-survey (2019)."},{"key":"516_CR10","unstructured":"Forrester Consulting. Overcome Obstacles To Get To AI At Scale. IBM https:\/\/www.ibm.com\/downloads\/cas\/VBMPEQLN (2020)."},{"key":"516_CR11","unstructured":"State of data science 2020. Anaconda https:\/\/www.anaconda.com\/state-of-data-science-2020 (2020)."},{"key":"516_CR12","doi-asserted-by":"publisher","first-page":"289","DOI":"10.1038\/nbt0418-289","volume":"36","author":"J Petrone","year":"2018","unstructured":"Petrone, J. Roche pays $1.9 billion for Flatiron\u2019s army of electronic health record curators. Nat. Biotechnol. 36, 289\u2013290 (2018).","journal-title":"Nat. Biotechnol."},{"key":"516_CR13","doi-asserted-by":"publisher","first-page":"665","DOI":"10.1038\/s42256-020-00257-z","volume":"2","author":"R Geirhos","year":"2020","unstructured":"Geirhos, R. et al. Shortcut learning in deep neural networks. Nat. Mach. Intell. 2, 665\u2013673 (2020).","journal-title":"Nat. Mach. Intell."},{"key":"516_CR14","unstructured":"Daneshjou, R. et al. Disparities in dermatology AI: assessments using diverse clinical images. Preprint at http:\/\/arxiv.org\/abs\/2111.08006 (2021)."},{"key":"516_CR15","unstructured":"Koch, B., Denton, E., Hanna, A. & Foster, J. G. Reduced, reused and recycled: the life of a dataset in machine learning research. In NeurIPS 2021 Datasets and Benchmarks Track 50 (OpenReview, 2021)."},{"key":"516_CR16","unstructured":"Coleman, C. et al. DAWNBench: An end-to-end deep learning benchmark and competition. In NeurIPS MLSys Workshop 10 (MLSys, 2017)."},{"key":"516_CR17","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1007\/s11263-016-0981-7","volume":"123","author":"R Krishna","year":"2017","unstructured":"Krishna, R. et al. Visual genome: connecting language and vision using crowdsourced dense image annotations. Int. J. Comput. Vision 123, 32\u201373 (2017).","journal-title":"Int. J. Comput. Vision"},{"key":"516_CR18","doi-asserted-by":"crossref","unstructured":"Kiela, D. et al. Dynabench: rethinking benchmarking in NLP. In Proc. 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies 4110\u20134124 (ACL, 2021).","DOI":"10.18653\/v1\/2021.naacl-main.324"},{"key":"516_CR19","doi-asserted-by":"publisher","unstructured":"Sambasivan, N. et al. \u2018Everyone wants to do the model work, not the data work\u2019: data cascades in high-stakes AI. In Proc. 2021 CHI Conference on Human Factors in Computing Systems (ACM, 2021); https:\/\/doi.org\/10.1145\/3411764.3445518","DOI":"10.1145\/3411764.3445518"},{"key":"516_CR20","doi-asserted-by":"publisher","first-page":"1362","DOI":"10.1001\/jamadermatol.2021.3129","volume":"157","author":"R Daneshjou","year":"2021","unstructured":"Daneshjou, R., Smith, M. P., Sun, M. D., Rotemberg, V. & Zou, J. Lack of transparency and potential bias in artificial intelligence data sets and algorithms: a scoping review. JAMA Dermatol. 157, 1362\u20131369 (2021).","journal-title":"JAMA Dermatol."},{"key":"516_CR21","doi-asserted-by":"publisher","first-page":"582","DOI":"10.1038\/s41591-021-01312-x","volume":"27","author":"E Wu","year":"2021","unstructured":"Wu, E. et al. How medical AI devices are evaluated: limitations and recommendations from an analysis of FDA approvals. Nat. Med. 27, 582\u2013584 (2021).","journal-title":"Nat. Med."},{"key":"516_CR22","doi-asserted-by":"publisher","first-page":"100336","DOI":"10.1016\/j.patter.2021.100336","volume":"2","author":"A Paullada","year":"2021","unstructured":"Paullada, A., Raji, I. D., Bender, E. M., Denton, E. & Hanna, A. Data and its (dis)contents: a survey of dataset development and use in machine learning research. Patterns 2, 100336 (2021).","journal-title":"Patterns"},{"key":"516_CR23","doi-asserted-by":"publisher","first-page":"559","DOI":"10.1038\/s41592-018-0083-2","volume":"15","author":"B Smucker","year":"2018","unstructured":"Smucker, B., Krzywinski, M. & Altman, N. Optimal experimental design. Nat. Methods 15, 559\u2013560 (2018).","journal-title":"Nat. Methods"},{"key":"516_CR24","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/978-3-031-01892-3","volume":"4","author":"W Fan","year":"2012","unstructured":"Fan, W. & Geerts, F. Foundations of data quality management. Synth. Lect. Data Manag. 4, 1\u2013217 (2012).","journal-title":"Synth. Lect. Data Manag."},{"key":"516_CR25","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3457607","volume":"54","author":"N Mehrabi","year":"2021","unstructured":"Mehrabi, N., Morstatter, F., Saxena, N., Lerman, K. & Galstyan, A. A survey on bias and fairness in machine learning. ACM Comput. Surv. 54, 1\u201335 (2021).","journal-title":"ACM Comput. Surv."},{"key":"516_CR26","unstructured":"Buolamwini, J. & Gebru, T. Gender shades: intersectional accuracy disparities in commercial gender classification. In Proc. 1st Conference on Fairness, Accountability and Transparency 77\u201391 (PMLR, 2018)."},{"key":"516_CR27","doi-asserted-by":"publisher","first-page":"1212","DOI":"10.1001\/jama.2020.12067","volume":"324","author":"A Kaushal","year":"2020","unstructured":"Kaushal, A., Altman, R. & Langlotz, C. Geographic distribution of US cohorts used to train deep learning algorithms. J. Am. Med. Assoc. 324, 1212\u20131213 (2020).","journal-title":"J. Am. Med. Assoc."},{"key":"516_CR28","doi-asserted-by":"publisher","first-page":"324","DOI":"10.1038\/d41586-018-05707-8","volume":"559","author":"J Zou","year":"2018","unstructured":"Zou, J. & Schiebinger, L. AI can be sexist and racist\u2014it\u2019s time to make it fair. Nature 559, 324\u2013326 (2018).","journal-title":"Nature"},{"key":"516_CR29","doi-asserted-by":"publisher","unstructured":"Coston, A. et al. Leveraging administrative data for bias audits: assessing disparate coverage with mobility data for COVID-19 policy. In Proc. 2021 ACM Conference on Fairness, Accountability, and Transparency 173\u2013184 (ACM, 2021); https:\/\/doi.org\/10.1145\/3442188.3445881","DOI":"10.1145\/3442188.3445881"},{"key":"516_CR30","unstructured":"Mozilla. Mozilla Common Voice receives $3.4 million investment to democratize and diversify voice tech in East Africa. Mozilla Foundation https:\/\/foundation.mozilla.org\/en\/blog\/mozilla-common-voice-receives-34-million-investment-to-democratize-and-diversify-voice-tech-in-east-africa\/ (2021)."},{"key":"516_CR31","unstructured":"Reid, K. Community partnerships and technical excellence unlock open voice technology success in Rwanda. Mozilla Foundation https:\/\/foundation.mozilla.org\/en\/blog\/open-voice-success-in-rwanda\/ (2021)."},{"key":"516_CR32","doi-asserted-by":"publisher","first-page":"354","DOI":"10.1038\/d41586-020-03187-3","volume":"587","author":"R Van Noorden","year":"2020","unstructured":"Van Noorden, R. The ethical questions that haunt facial-recognition research. Nature 587, 354\u2013358 (2020).","journal-title":"Nature"},{"key":"516_CR33","unstructured":"Build more ethical AI. Synthesis AI https:\/\/synthesis.ai\/use-cases\/bias-reduction\/ (2022)."},{"key":"516_CR34","doi-asserted-by":"crossref","unstructured":"Kortylewski, A. et al. Analyzing and reducing the damage of dataset bias to face recognition with synthetic data. In IEEE Conference on Computer Vision and Pattern Recognition Workshops 2261\u20132268 (IEEE, 2019).","DOI":"10.1109\/CVPRW.2019.00279"},{"key":"516_CR35","doi-asserted-by":"crossref","unstructured":"Nikolenko, S. I. Synthetic Data for Deep Learning Vol. 174 (Springer, 2021).","DOI":"10.1007\/978-3-030-75178-4"},{"key":"516_CR36","unstructured":"Srivastava, S. et al. BEHAVIOR: Benchmark for Everyday Household Activities in Virtual, Interactive, and Ecological Environments. In Proc. 5th Annual Conference on Robot Learning Vol. 164 477\u2013490 (PMLR, 2022)."},{"key":"516_CR37","unstructured":"Li, C. et al. iGibson 2.0: object-centric simulation for robot learning of everyday household tasks. In Proc. 5th Annual Conference on Robot Learning Vol. 164 455\u2013465 (PMLR, 2022)."},{"key":"516_CR38","unstructured":"H\u00f6fer, S. et al. Perspectives on Sim2Real transfer for robotics: a summary of the R:SS 2020 workshop. Preprint at http:\/\/arxiv.org\/abs\/2012.03806 (2020)"},{"key":"516_CR39","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3395208","volume":"39","author":"B Egger","year":"2020","unstructured":"Egger, B. et al. 3D morphable face models\u2014past, present, and future. ACM Trans. Graph. 39, 1\u201338 (2020).","journal-title":"ACM Trans. Graph."},{"key":"516_CR40","first-page":"1887","volume":"119","author":"K Choi","year":"2020","unstructured":"Choi, K., Grover, A., Singh, T., Shu, R. & Ermon, S. Fair generative modeling via weak supervision. Proc. Mach. Learn. Res. 119, 1887\u20131898 (2020).","journal-title":"Proc. Mach. Learn. Res."},{"key":"516_CR41","unstructured":"Holland, S., Hosny, A., Newman, S., Joseph, J. & Chmielinski, K. The dataset nutrition label: a framework to drive higher data quality standards. Preprint at https:\/\/arxiv.org\/abs\/1805.03677 (2018)."},{"key":"516_CR42","doi-asserted-by":"publisher","first-page":"86","DOI":"10.1145\/3458723","volume":"64","author":"T Gebru","year":"2021","unstructured":"Gebru, T. et al. Datasheets for datasets. Commun. ACM 64, 86\u201392 (2021).","journal-title":"Commun. ACM"},{"key":"516_CR43","doi-asserted-by":"publisher","first-page":"587","DOI":"10.1162\/tacl_a_00041","volume":"6","author":"EM Bender","year":"2018","unstructured":"Bender, E. M. & Friedman, B. Data statements for natural language processing: toward mitigating system bias and enabling better science. Trans. Assoc. Comput. Linguist. 6, 587\u2013604 (2018).","journal-title":"Trans. Assoc. Comput. Linguist."},{"key":"516_CR44","doi-asserted-by":"crossref","unstructured":"Wang, A., Narayanan, A. & Russakovsky, O. REVISE: a tool for measuring and mitigating bias in visual datasets. In European Conference on Computer Vision 733\u2013751 (Springer, 2020).","DOI":"10.1007\/978-3-030-58580-8_43"},{"key":"516_CR45","doi-asserted-by":"crossref","unstructured":"Miceli, M. et al. Documenting computer vision datasets: an invitation to reflexive data practices. In Proc. 2021 ACM on Conference on Fairness, Accountability, and Transparency 161\u2013172 (2021).","DOI":"10.1145\/3442188.3445880"},{"key":"516_CR46","doi-asserted-by":"publisher","first-page":"317:1","DOI":"10.1145\/3476058","volume":"5","author":"MK Scheuerman","year":"2021","unstructured":"Scheuerman, M. K., Hanna, A. & Denton, E. Do datasets have politics? Disciplinary values in computer vision dataset development. Proc. ACM Hum. Comput. Interact. 5, 317:1\u2013317:37 (2021).","journal-title":"Proc. ACM Hum. Comput. Interact."},{"key":"516_CR47","unstructured":"Liang, W. & Zou, J. MetaShift: a dataset of datasets for evaluating contextual distribution shifts and training conflicts. In International Conference on Learning Representations 400 (OpenReview, 2022)."},{"key":"516_CR48","first-page":"2242","volume":"97","author":"A Ghorbani","year":"2019","unstructured":"Ghorbani, A. & Zou, J. Data Shapley: equitable valuation of data for machine learning. Proc. Mach. Learn. Res. 97, 2242\u20132251 (2019).","journal-title":"Proc. Mach. Learn. Res."},{"key":"516_CR49","first-page":"793","volume":"130","author":"Y Kwon","year":"2021","unstructured":"Kwon, Y., Rivas, M. A. & Zou, J. Efficient computation and analysis of distributional Shapley values. Proc. Mach. Learn. Res. 130, 793\u2013801 (2021).","journal-title":"Proc. Mach. Learn. Res."},{"key":"516_CR50","first-page":"1167","volume":"89","author":"R Jia","year":"2019","unstructured":"Jia, R. et al. Towards efficient data valuation based on the Shapley value. Proc. Mach. Learn. Res. 89, 1167\u20131176 (2019).","journal-title":"Proc. Mach. Learn. Res."},{"key":"516_CR51","first-page":"1885","volume":"70","author":"PW Koh","year":"2017","unstructured":"Koh, P. W. & Liang, P. Understanding black-box predictions via influence functions. Proc. Mach. Learn. Res. 70, 1885\u20131894 (2017).","journal-title":"Proc. Mach. Learn. Res."},{"key":"516_CR52","unstructured":"Kwon, Y. & Zou, J. Beta Shapley: a unified and noise-reduced data valuation framework for machine learning. In Proc. 25th International Conference on Artificial Intelligence and Statistics Vol. 151 8780\u20138802 (PMLR, 2022)."},{"key":"516_CR53","doi-asserted-by":"publisher","first-page":"1373","DOI":"10.1613\/jair.1.12125","volume":"70","author":"C Northcutt","year":"2021","unstructured":"Northcutt, C., Jiang, L. & Chuang, I. Confident learning: estimating uncertainty in dataset labels. J. Artif. Intell. Res. 70, 1373\u20131411 (2021).","journal-title":"J. Artif. Intell. Res."},{"key":"516_CR54","unstructured":"Northcutt, C. G., Athalye, A. & Mueller, J. Pervasive label errors in test sets destabilize machine learning benchmarks. In NeurIPS 2021 Datasets and Benchmarks Track 172 (OpenReview, 2021)."},{"key":"516_CR55","doi-asserted-by":"crossref","unstructured":"Dodge, J. et al. Documenting large webtext corpora: a case study on the Colossal Clean Crawled Corpus. In Proc. 2021 Conference on Empirical Methods in Natural Language Processing 12861305 (ACL, 2021).","DOI":"10.18653\/v1\/2021.emnlp-main.98"},{"key":"516_CR56","doi-asserted-by":"publisher","first-page":"948","DOI":"10.14778\/2994509.2994514","volume":"9","author":"S Krishnan","year":"2016","unstructured":"Krishnan, S., Wang, J., Wu, E., Franklin, M. J. & Goldberg, K. ActiveClean: interactive data cleaning for statistical modeling. Proc. VLDB Endow. 9, 948\u2013959 (2016).","journal-title":"Proc. VLDB Endow."},{"key":"516_CR57","unstructured":"Rolnick, D., Veit, A., Belongie, S. & Shavit, N. Deep learning is robust to massive label noise. Preprint at http:\/\/arxiv.org\/abs\/1705.10694 (2018)."},{"key":"516_CR58","doi-asserted-by":"publisher","unstructured":"Geiger, A., Lenz, P. & Urtasun, R. Are we ready for autonomous driving? The KITTI vision benchmark suite. In 2012 IEEE Conference on Computer Vision and Pattern Recognition 3354\u20133361 (IEEE, 2012); https:\/\/doi.org\/10.1109\/CVPR.2012.6248074","DOI":"10.1109\/CVPR.2012.6248074"},{"key":"516_CR59","doi-asserted-by":"crossref","unstructured":"Sun, P. et al. Scalability in perception for autonomous driving: Waymo Open Dataset. In Proc. IEEE\/CVF Conference on Computer Vision and Pattern Recognition 2446\u20132454 (IEEE, 2020).","DOI":"10.1109\/CVPR42600.2020.00252"},{"key":"516_CR60","first-page":"115","volume":"7","author":"J Park","year":"2019","unstructured":"Park, J., Krishna, R., Khadpe, P., Fei-Fei, L. & Bernstein, M. AI-based request augmentation to increase crowdsourcing participation. Proc. AAAI Conf. Hum. Comput. Crowdsourcing 7, 115\u2013124 (2019).","journal-title":"Proc. AAAI Conf. Hum. Comput. Crowdsourcing"},{"key":"516_CR61","doi-asserted-by":"publisher","first-page":"709","DOI":"10.1007\/s00778-019-00552-1","volume":"29","author":"A Ratner","year":"2020","unstructured":"Ratner, A. et al. Snorkel: rapid training data creation with weak supervision. VLDB J. 29, 709\u2013730 (2020).","journal-title":"VLDB J."},{"key":"516_CR62","first-page":"3567","volume":"29","author":"AJ Ratner","year":"2016","unstructured":"Ratner, A. J., De, Sa,C. M., Wu, S., Selsam, D. & R\u00e9, C. Data programming: creating large training sets, quickly. Adv. Neural Inf. Process. Syst. 29, 3567\u20133575 (2016).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"516_CR63","doi-asserted-by":"crossref","unstructured":"Liang, W., Liang, K.-H. & Yu, Z. HERALD: an annotation efficient method to detect user disengagement in social conversations. In Proc. 59th Annual Meeting of the Association for Computational Linguistics 3652\u20133665 (ACL, 2021).","DOI":"10.18653\/v1\/2021.acl-long.283"},{"key":"516_CR64","unstructured":"Settles, B. Active Learning Literature Survey. MINDS@UW http:\/\/digital.library.wisc.edu\/1793\/60660 (University of Wisconsin-Madison, 2009)."},{"key":"516_CR65","doi-asserted-by":"crossref","unstructured":"Coleman, C. et al. Similarity search for efficient active learning and search of rare concepts. In Proc. AAAI Conference on Artificial Intelligence Vol. 36 6402\u20136410 (2022).","DOI":"10.1609\/aaai.v36i6.20591"},{"key":"516_CR66","doi-asserted-by":"crossref","unstructured":"Liang, W., Zou, J. & Yu, Z. ALICE: Active Learning with Contrastive Natural Language Explanations. In Proc. 2020 Conference on Empirical Methods in Natural Language Processing 4380\u20134391 (ACL, 2020).","DOI":"10.18653\/v1\/2020.emnlp-main.355"},{"key":"516_CR67","doi-asserted-by":"crossref","unstructured":"Hollenstein, N. & Zhang, C. Entity recognition at first sight: improving NER with eye movement information. In Proc. 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies 1\u201310 (ACL, 2019).","DOI":"10.18653\/v1\/N19-1001"},{"key":"516_CR68","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-020-18360-5","volume":"11","author":"N Valliappan","year":"2020","unstructured":"Valliappan, N. et al. Accelerating eye movement research via accurate and affordable smartphone eye tracking. Nat. Commun. 11, 4553 (2020).","journal-title":"Nat. Commun."},{"key":"516_CR69","doi-asserted-by":"crossref","unstructured":"Saab, K. et al. Observational supervision for medical image classification using gaze data. In International Conference on Medical Image Computing and Computer-Assisted Intervention 603\u2013614 (Springer, 2021).","DOI":"10.1007\/978-3-030-87196-3_56"},{"key":"516_CR70","unstructured":"Kang, D., Raghavan, D., Bailis, P. & Zaharia, M. Model assertions for debugging machine learning. In NeurIPS MLSys Workshop 23 (MLSys, 2020)."},{"key":"516_CR71","first-page":"1097","volume":"25","author":"A Krizhevsky","year":"2012","unstructured":"Krizhevsky, A., Sutskever, I. & Hinton, G. E. Imagenet classification with deep convolutional neural networks. Adv. Neural Inf. Process. Syst. 25, 1097\u20131105 (2012).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"516_CR72","doi-asserted-by":"crossref","unstructured":"Sennrich, R., Haddow, B. & Birch, A. Improving neural machine translation models with monolingual data. In Proc. 54th Annual Meeting of the Association for Computational Linguistics 86\u201396 (ACL, 2016).","DOI":"10.18653\/v1\/P16-1009"},{"key":"516_CR73","unstructured":"Zhang, H., Ciss\u00e9, M., Dauphin, Y. N. & Lopez-Paz, D. mixup: beyond empirical risk minimization. In Proc. International Conference on Learning Representations 296 (OpenReview, 2018)."},{"key":"516_CR74","doi-asserted-by":"publisher","unstructured":"Liang, W. & Zou, J. Neural group testing to accelerate deep learning. In 2021 IEEE International Symposium on Information Theory (ISIT) 958\u2013963 (IEEE, 2021); https:\/\/doi.org\/10.1109\/ISIT45174.2021.9518038","DOI":"10.1109\/ISIT45174.2021.9518038"},{"key":"516_CR75","doi-asserted-by":"crossref","unstructured":"Cubuk, E. D., Zoph, B., Shlens, J. & Le, Q. V. Randaugment: practical automated data augmentation with a reduced search space. In Proc. IEEE\/CVF Conference on Computer Vision and Pattern Recognition Workshops 702\u2013703 (IEEE, 2020).","DOI":"10.1109\/CVPRW50498.2020.00359"},{"key":"516_CR76","doi-asserted-by":"crossref","unstructured":"Caron, M., Bojanowski, P., Joulin, A. & Douze, M. Deep clustering for unsupervised learning of visual features. In Proc. European Conference on Computer Vision (ECCV) 132\u2013149 (2018).","DOI":"10.1007\/978-3-030-01264-9_9"},{"key":"516_CR77","first-page":"2845","volume":"130","author":"Z Deng","year":"2021","unstructured":"Deng, Z., Zhang, L., Ghorbani, A. & Zou, J. Improving adversarial robustness via unlabeled out-of-domain. Data. Proc. Mach. Learn. Res. 130, 2845\u20132853 (2021).","journal-title":"Data. Proc. Mach. Learn. Res."},{"key":"516_CR78","unstructured":"Zhang, L., Deng, Z., Kawaguchi, K., Ghorbani, A. & Zou, J. How does mixup help with robustness and generalization? In Proc. International Conference on Learning Representations 79 (OpenReview, 2021)."},{"key":"516_CR79","doi-asserted-by":"publisher","first-page":"e1002683","DOI":"10.1371\/journal.pmed.1002683","volume":"15","author":"JR Zech","year":"2018","unstructured":"Zech, J. R. et al. Variable generalization performance of a deep learning model to detect pneumonia in chest radiographs: a cross-sectional study. PLoS Med. 15, e1002683 (2018).","journal-title":"PLoS Med."},{"key":"516_CR80","doi-asserted-by":"crossref","unstructured":"Gururangan, S. et al. Annotation artifacts in natural language inference data. In Proc. 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies 107\u2013112 (ACL, 2018).","DOI":"10.18653\/v1\/N18-2017"},{"key":"516_CR81","doi-asserted-by":"publisher","first-page":"103613","DOI":"10.1016\/j.ebiom.2021.103613","volume":"73","author":"JW Hughes","year":"2021","unstructured":"Hughes, J. W. et al. Deep learning evaluation of biomarkers from echocardiogram videos. EBioMedicine 73, 103613 (2021).","journal-title":"EBioMedicine"},{"key":"516_CR82","doi-asserted-by":"publisher","first-page":"137","DOI":"10.1038\/s41586-019-1657-6","volume":"575","author":"C Tannenbaum","year":"2019","unstructured":"Tannenbaum, C., Ellis, R. P., Eyssel, F., Zou, J. & Schiebinger, L. Sex and gender analysis improves science and engineering. Nature 575, 137\u2013146 (2019).","journal-title":"Nature"},{"key":"516_CR83","doi-asserted-by":"publisher","unstructured":"Kim, M. P., Ghorbani, A. & Zou, J. Y. Multiaccuracy: black-box post-processing for fairness in classification. In Proc. 2019 AAAI\/ACM Conference on AI, Ethics, and Society 247\u2013254 (ACM, 2019); https:\/\/doi.org\/10.1145\/3306618.3314287","DOI":"10.1145\/3306618.3314287"},{"key":"516_CR84","unstructured":"Eyuboglu, S. et al. Domino: discovering systematic errors with cross-modal embeddings. In Proc. International Conference on Learning Representations 1 (OpenReview, 2022)."},{"key":"516_CR85","doi-asserted-by":"publisher","unstructured":"Karla\u0161, B. et al. Building continuous integration services for machine learning. In Proc. 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining 2407\u20132415 (ACM, 2020); https:\/\/doi.org\/10.1145\/3394486.3403290","DOI":"10.1145\/3394486.3403290"},{"key":"516_CR86","unstructured":"Lambert, F. Tesla is collecting insane amount of data from its full self-driving test fleet. Electrek https:\/\/electrek.co\/2020\/10\/24\/tesla-collecting-insane-amount-data-full-self-driving-test-fleet\/ (2020)."},{"key":"516_CR87","unstructured":"Azizzadenesheli, K., Liu, A., Yang, F. & Anandkumar, A. Regularized learning for domain adaptation under label shifts. In Proc. International Conference on Learning Representations 432 (OpenReview, 2019)."},{"key":"516_CR88","doi-asserted-by":"publisher","unstructured":"Baylor, D. et al. TFX: a TensorFlow-based production-scale machine learning platform. In Proc. 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining 1387\u20131395 (ACM, 2017); https:\/\/doi.org\/10.1145\/3097983.3098021","DOI":"10.1145\/3097983.3098021"},{"key":"516_CR89","first-page":"39","volume":"41","author":"M Zaharia","year":"2018","unstructured":"Zaharia, M. et al. Accelerating the machine learning lifecycle with MLflow. IEEE Data Eng Bull 41, 39\u201345 (2018).","journal-title":"IEEE Data Eng Bull"},{"key":"516_CR90","unstructured":"Proposal for a Regulation of the European Parliament and the Council Laying down Harmonised Rules on Artificial Intelligence (Artificial Intelligence Act) and Amending Certain Union Legislative Acts COM(2021) 206 final (European Commission, 2021); https:\/\/eur-lex.europa.eu\/legal-content\/EN\/TXT\/HTML\/?uri=CELEX:52021PC0206&from=EN"},{"key":"516_CR91","doi-asserted-by":"publisher","first-page":"150","DOI":"10.1126\/science.aaz7028","volume":"367","author":"MM Mello","year":"2020","unstructured":"Mello, M. M., Triantis, G., Stanton, R., Blumenkranz, E. & Studdert, D. M. Waiting for data: barriers to executing data use agreements. Science 367, 150\u2013152 (2020).","journal-title":"Science"},{"key":"516_CR92","doi-asserted-by":"crossref","unstructured":"Andrus, M., Spitzer, E., Brown, J. & Xiang, A. What we can\u2019t measure, we can\u2019t understand: challenges to demographic data procurement in the pursuit of fairness. In Proc. 2021 ACM Conference on Fairness, Accountability, and Transparency 249\u2013260 (ACM, 2021).","DOI":"10.1145\/3442188.3445888"},{"key":"516_CR93","doi-asserted-by":"publisher","first-page":"1111","DOI":"10.1001\/archfami.9.10.1111","volume":"9","author":"SH Woolf","year":"2000","unstructured":"Woolf, S. H., Rothemich, S. F., Johnson, R. E. & Marsland, D. W. Selection bias from requiring patients to give consent to examine data for health services research. Arch. Fam. Med. 9, 1111\u20131118 (2000).","journal-title":"Arch. Fam. Med."},{"key":"516_CR94","doi-asserted-by":"publisher","first-page":"777","DOI":"10.1126\/science.275.5301.777","volume":"275","author":"E Marshall","year":"1997","unstructured":"Marshall, E. Is data-hoarding slowing the assault of pathogens? Science 275, 777\u2013780 (1997).","journal-title":"Science"},{"key":"516_CR95","doi-asserted-by":"crossref","unstructured":"Baeza-Yates, R. Data and algorithmic bias in the web. In Proc. 8th ACM Conference on Web Science 1 (ACM, 2016).","DOI":"10.1145\/2908131.2908135"},{"key":"516_CR96","doi-asserted-by":"publisher","first-page":"663","DOI":"10.1038\/gim.2015.138","volume":"18","author":"NA Garrison","year":"2016","unstructured":"Garrison, N. A. et al. A systematic literature review of individuals\u2019 perspectives on broad consent and data sharing in the United States. Genet. Med. 18, 663\u2013671 (2016).","journal-title":"Genet. Med."},{"key":"516_CR97","doi-asserted-by":"publisher","first-page":"194","DOI":"10.1038\/d41586-018-06948-3","volume":"562","author":"N Cox","year":"2018","unstructured":"Cox, N. UK Biobank shares the promise of big data. Nature 562, 194\u2013195 (2018).","journal-title":"Nature"},{"key":"516_CR98","unstructured":"Art. 20 GDPR: Right to Data Portability https:\/\/gdpr-info.eu\/art-20-gdpr\/ (General Data Protection Regulation, 2021)."},{"key":"516_CR99","unstructured":"TITLE 1.81.5. California Consumer Privacy Act of 2018 https:\/\/leginfo.legislature.ca.gov\/faces\/codes_displayText.xhtml?division=3.&part=4.&lawCode=CIV&title=1.81.5 (California Legislative Information, 2018)."},{"key":"516_CR100","unstructured":"Kr\u00e4mer, J., Senellart, P. & de Streel, A. Making Data Portability More Effective for the Digital Economy: Economic Implications and Regulatory Challenges (CERRE, 2020)."},{"key":"516_CR101","unstructured":"Loh, W., Hauschke, A., Puntschuh, M. & Hallensleben, S. VDE SPEC 90012 V1.0: VCIO Based Description of Systems for AI Trustworthiness Characterisation (VDE Press, 2022)."},{"key":"516_CR102","unstructured":"Can artificial intelligence conform to values? VDE SPEC as the basis for future developments. VDE Presse https:\/\/www.vde.com\/ai-trust (2022)."},{"key":"516_CR103","doi-asserted-by":"crossref","unstructured":"Mitchell, M. et al. Model cards for model reporting. In Proc. Conference on Fairness, Accountability, and Transparency 220\u2013229 (ACM, 2019).","DOI":"10.1145\/3287560.3287596"},{"key":"516_CR104","first-page":"15453","volume":"32","author":"E Bagdasaryan","year":"2019","unstructured":"Bagdasaryan, E., Poursaeed, O. & Shmatikov, V. Differential privacy has disparate impact on model accuracy. Adv. Neural Inf. Process. Syst. 32, 15453\u201315462 (2019).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"516_CR105","doi-asserted-by":"crossref","unstructured":"Lyu, L., Yu, H. & Yang, Q. Threats to federated learning: a survey. Preprint at http:\/\/arxiv.org\/abs\/2003.02133 (2020).","DOI":"10.1007\/978-3-030-63076-8_1"},{"key":"516_CR106","first-page":"2008","volume":"130","author":"Z Izzo","year":"2021","unstructured":"Izzo, Z., Smart, M. A., Chaudhuri, K. & Zou, J. Approximate data deletion from machine learning models. Proc. Mach. Learn. Res. 130, 2008\u20132016 (2021).","journal-title":"Proc. Mach. Learn. Res."},{"key":"516_CR107","doi-asserted-by":"publisher","first-page":"33","DOI":"10.1287\/mksc.2019.1198","volume":"39","author":"GA Johnson","year":"2020","unstructured":"Johnson, G. A., Shriver, S. K. & Du, S. Consumer privacy choice in online advertising: who opts out and at what cost to industry? Mark. Sci. 39, 33\u201351 (2020).","journal-title":"Mark. Sci."},{"key":"516_CR108","doi-asserted-by":"publisher","unstructured":"Wilson, D. R. Beyond probabilistic record linkage: Using neural networks and complex features to improve genealogical record linkage. In 2011 International Joint Conference on Neural Networks 9\u201314 (IEEE, 2011); https:\/\/doi.org\/10.1109\/IJCNN.2011.6033192","DOI":"10.1109\/IJCNN.2011.6033192"},{"key":"516_CR109","doi-asserted-by":"publisher","unstructured":"Kallus, N., Mao, X. & Zhou, A. Assessing algorithmic fairness with unobserved protected class using data combination. Manag. Sci. https:\/\/doi.org\/10.1287\/mnsc.2020.3850 (2021).","DOI":"10.1287\/mnsc.2020.3850"},{"key":"516_CR110","doi-asserted-by":"crossref","unstructured":"Deng, J. et al. Imagenet: a large-scale hierarchical image database. In 2009 IEEE Conference on Computer Vision and Pattern Recognition 248\u2013255 (IEEE, 2009).","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"516_CR111","doi-asserted-by":"publisher","unstructured":"Yang, K., Qinami, K., Fei-Fei, L., Deng, J. & Russakovsky, O. Towards fairer datasets: filtering and balancing the distribution of the people subtree in the ImageNet hierarchy. In Proc. 2020 Conference on Fairness, Accountability, and Transparency 547\u2013558 (ACM, 2020); https:\/\/doi.org\/10.1145\/3351095.3375709","DOI":"10.1145\/3351095.3375709"},{"key":"516_CR112","unstructured":"DCBench: a benchmark of data-centric tasks from across the machine learning lifecycle. DCAI https:\/\/www.datacentricai.cc\/benchmark\/ (2021)."},{"key":"516_CR113","doi-asserted-by":"publisher","unstructured":"Zaugg, I. A., Hossain, A. & Molloy, B. Digitally-disadvantaged languages. Internet Policy Rev. https:\/\/doi.org\/10.14763\/2022.2.1654 (2022).","DOI":"10.14763\/2022.2.1654"},{"key":"516_CR114","unstructured":"Victor, D. COCO-Africa: a curation tool and dataset of common objects in the context of Africa. In 2018 Conference on Neural Information Processing, 2nd Black in AI Workshop 1 (NeurIPS, 2019)."},{"key":"516_CR115","doi-asserted-by":"publisher","first-page":"1116","DOI":"10.1162\/tacl_a_00416","volume":"9","author":"DI Adelani","year":"2021","unstructured":"Adelani, D. I. et al. MasakhaNER: Named Entity Recognition for African languages. Trans. Assoc. Comput. Linguist. 9, 1116\u20131131 (2021).","journal-title":"Trans. Assoc. Comput. Linguist."},{"key":"516_CR116","unstructured":"Siminyu, K. et al. AI4D\u2014African language program. Preprint at http:\/\/arxiv.org\/abs\/2104.02516 (2021)."},{"key":"516_CR117","doi-asserted-by":"publisher","first-page":"101034","DOI":"10.1016\/j.eclinm.2021.101034","volume":"38","author":"G Frija","year":"2021","unstructured":"Frija, G. et al. How to improve access to medical imaging in low- and middle-income countries? EClinicalMedicine 38, 101034 (2021).","journal-title":"EClinicalMedicine"}],"updated-by":[{"DOI":"10.1038\/s42256-022-00548-7","type":"correction","label":"Correction","source":"publisher","updated":{"date-parts":[[2022,9,21]],"date-time":"2022-09-21T00:00:00Z","timestamp":1663718400000}}],"container-title":["Nature Machine Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s42256-022-00516-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s42256-022-00516-1","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s42256-022-00516-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,11,25]],"date-time":"2022-11-25T09:39:05Z","timestamp":1669369145000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s42256-022-00516-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,8,17]]},"references-count":117,"journal-issue":{"issue":"8","published-online":{"date-parts":[[2022,8]]}},"alternative-id":["516"],"URL":"https:\/\/doi.org\/10.1038\/s42256-022-00516-1","relation":{"correction":[{"id-type":"doi","id":"10.1038\/s42256-022-00548-7","asserted-by":"object"}]},"ISSN":["2522-5839"],"issn-type":[{"value":"2522-5839","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,8,17]]},"assertion":[{"value":"3 April 2022","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 June 2022","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 August 2022","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"21 September 2022","order":4,"name":"change_date","label":"Change Date","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"Correction","order":5,"name":"change_type","label":"Change Type","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"A Correction to this paper has been published:","order":6,"name":"change_details","label":"Change Details","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"https:\/\/doi.org\/10.1038\/s42256-022-00548-7","URL":"https:\/\/doi.org\/10.1038\/s42256-022-00548-7","order":7,"name":"change_details","label":"Change Details","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"M.Z. is a co-founder of Databricks. The other authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}