{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,16]],"date-time":"2026-04-16T01:10:04Z","timestamp":1776301804758,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":129,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,5,6]],"date-time":"2021-05-06T00:00:00Z","timestamp":1620259200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-sa\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,5,6]]},"DOI":"10.1145\/3411764.3445518","type":"proceedings-article","created":{"date-parts":[[2021,5,8]],"date-time":"2021-05-08T02:27:22Z","timestamp":1620440842000},"page":"1-15","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":516,"title":["\u201cEveryone wants to do the model work, not the data work\u201d: Data Cascades in High-Stakes AI"],"prefix":"10.1145","author":[{"given":"Nithya","family":"Sambasivan","sequence":"first","affiliation":[{"name":"Google Research, United States"}]},{"given":"Shivani","family":"Kapania","sequence":"additional","affiliation":[{"name":"Google Research India, India"}]},{"given":"Hannah","family":"Highfill","sequence":"additional","affiliation":[{"name":"Google Inc., United States"}]},{"given":"Diana","family":"Akrong","sequence":"additional","affiliation":[{"name":"Google Research Accra, Ghana"}]},{"given":"Praveen","family":"Paritosh","sequence":"additional","affiliation":[{"name":"Google, United States"}]},{"given":"Lora M","family":"Aroyo","sequence":"additional","affiliation":[{"name":"Google, United States"}]}],"member":"320","published-online":{"date-parts":[[2021,5,7]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"[n.d.]. 2019 Kaggle ML & DS Survey | Kaggle. https:\/\/www.kaggle.com\/c\/kaggle-survey-2019. (Accessed on 08\/29\/2020).  [n.d.]. 2019 Kaggle ML & DS Survey | Kaggle. https:\/\/www.kaggle.com\/c\/kaggle-survey-2019. (Accessed on 08\/29\/2020)."},{"key":"e_1_3_2_1_2_1","unstructured":"[n.d.]. AI Readiness Index 2019 | AI4D | IAPD. https:\/\/ai4d.ai\/index2019\/. (Accessed on 09\/14\/2020).  [n.d.]. AI Readiness Index 2019 | AI4D | IAPD. https:\/\/ai4d.ai\/index2019\/. (Accessed on 09\/14\/2020)."},{"key":"e_1_3_2_1_3_1","unstructured":"[n.d.]. Landscape of AI-ML Research in India. http:\/\/www.itihaasa.com\/pdf\/Report_Final_ES.pdf. (Accessed on 09\/15\/2020).  [n.d.]. Landscape of AI-ML Research in India. http:\/\/www.itihaasa.com\/pdf\/Report_Final_ES.pdf. (Accessed on 09\/15\/2020)."},{"key":"e_1_3_2_1_4_1","unstructured":"[n.d.]. UCI Machine Learning Repository. https:\/\/archive.ics.uci.edu\/ml\/index.php. (Accessed on 09\/15\/2020).  [n.d.]. UCI Machine Learning Repository. https:\/\/archive.ics.uci.edu\/ml\/index.php. (Accessed on 09\/15\/2020)."},{"key":"e_1_3_2_1_5_1","unstructured":"[n.d.]. A Vision of AI for Joyful Education - Scientific American Blog Network. https:\/\/blogs.scientificamerican.com\/observations\/a-vision-of-ai-for-joyful-education\/. (Accessed on 09\/14\/2020).  [n.d.]. A Vision of AI for Joyful Education - Scientific American Blog Network. https:\/\/blogs.scientificamerican.com\/observations\/a-vision-of-ai-for-joyful-education\/. (Accessed on 09\/14\/2020)."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v35i4.2513"},{"key":"e_1_3_2_1_8_1","unstructured":"Dario Amodei Chris Olah Jacob Steinhardt Paul Christiano John Schulman and Dan Man\u00e9. 2016. Concrete problems in AI safety. (2016). arXiv:1606.06565  Dario Amodei Chris Olah Jacob Steinhardt Paul Christiano John Schulman and Dan Man\u00e9. 2016. Concrete problems in AI safety. (2016). arXiv:1606.06565"},{"key":"e_1_3_2_1_9_1","unstructured":"Appen. 2020. The 2020 Machine Learning Report and State of AI. https:\/\/appen.com\/whitepapers\/the-state-of-ai-and-machine-learning-report\/. (Accessed on 09\/16\/2020).  Appen. 2020. The 2020 Machine Learning Report and State of AI. https:\/\/appen.com\/whitepapers\/the-state-of-ai-and-machine-learning-report\/. (Accessed on 09\/16\/2020)."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308560.3317083"},{"key":"e_1_3_2_1_11_1","unstructured":"[\n  11\n  ]  Lora Aroyo Anca Dumitrache Jennimaria Palomaki Praveen Paritosh Alex Quinn Olivia Rhinehart Mike Schaekermann Michael Tseng and Chris Welty.[n.d.]. https:\/\/sadworkshop.wordpress.com\/  [11] Lora Aroyo Anca Dumitrache Jennimaria Palomaki Praveen Paritosh Alex Quinn Olivia Rhinehart Mike Schaekermann Michael Tseng and Chris Welty.[n.d.]. https:\/\/sadworkshop.wordpress.com\/"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.15346\/hc.v1i1.34"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v36i1.2564"},{"key":"e_1_3_2_1_14_1","unstructured":"Jonathan Bailey. 2019. Why Siraj Raval\u2019s Plagiarism is the Future of Plagiarism - Plagiarism Today. https:\/\/www.plagiarismtoday.com\/2019\/10\/16\/why-siraj-ravals-plagiarism-is-the-future-of-plagiarism\/. (Accessed on 09\/15\/2020).  Jonathan Bailey. 2019. Why Siraj Raval\u2019s Plagiarism is the Future of Plagiarism - Plagiarism Today. https:\/\/www.plagiarismtoday.com\/2019\/10\/16\/why-siraj-ravals-plagiarism-is-the-future-of-plagiarism\/. (Accessed on 09\/15\/2020)."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v7i1.5285"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1177\/2053951718819569"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376718"},{"key":"e_1_3_2_1_18_1","unstructured":"Yoshua Bengio. 2020. Time to rethink the publication process in machine learning - Yoshua Bengio. https:\/\/yoshuabengio.org\/2020\/02\/26\/time-to-rethink-the-publication-process-in-machine-learning\/. (Accessed on 08\/18\/2020).  Yoshua Bengio. 2020. Time to rethink the publication process in machine learning - Yoshua Bengio. https:\/\/yoshuabengio.org\/2020\/02\/26\/time-to-rethink-the-publication-process-in-machine-learning\/. (Accessed on 08\/18\/2020)."},{"key":"e_1_3_2_1_19_1","volume-title":"Datahub: Collaborative data science & dataset version management at scale.","author":"Bhardwaj Anant","year":"2014","unstructured":"Anant Bhardwaj , Souvik Bhattacherjee , Amit Chavan , Amol Deshpande , Aaron\u00a0 J Elmore , Samuel Madden , and Aditya\u00a0 G Parameswaran . 2014 . Datahub: Collaborative data science & dataset version management at scale. (2014). arXiv:1409.0798 Anant Bhardwaj, Souvik Bhattacherjee, Amit Chavan, Amol Deshpande, Aaron\u00a0J Elmore, Samuel Madden, and Aditya\u00a0G Parameswaran. 2014. Datahub: Collaborative data science & dataset version management at scale. (2014). arXiv:1409.0798"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"Joshua Blumenstock. 2018. Don\u2019t forget people in the use of big data for development.  Joshua Blumenstock. 2018. Don\u2019t forget people in the use of big data for development.","DOI":"10.1038\/d41586-018-06215-5"},{"key":"e_1_3_2_1_21_1","volume-title":"Conference on Systems and Machine Learning (SysML). https:\/\/www. sysml. cc\/doc\/2019\/167","author":"Breck Eric","year":"2019","unstructured":"Eric Breck , Neoklis Polyzotis , Sudip Roy , Steven\u00a0Euijong Whang , and Martin Zinkevich . 2019 . Data validation for machine learning . In Conference on Systems and Machine Learning (SysML). https:\/\/www. sysml. cc\/doc\/2019\/167 . pdf. Eric Breck, Neoklis Polyzotis, Sudip Roy, Steven\u00a0Euijong Whang, and Martin Zinkevich. 2019. Data validation for machine learning. In Conference on Systems and Machine Learning (SysML). https:\/\/www. sysml. cc\/doc\/2019\/167. pdf."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3378393.3402278"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-44503-X_20"},{"key":"e_1_3_2_1_24_1","unstructured":"Andrew Burt and Patrick Hall. 2020. What to Do When AI Fails \u2013 O\u2019Reilly. https:\/\/www.oreilly.com\/radar\/what-to-do-when-ai-fails\/. (Accessed on 09\/16\/2020).  Andrew Burt and Patrick Hall. 2020. What to Do When AI Fails \u2013 O\u2019Reilly. https:\/\/www.oreilly.com\/radar\/what-to-do-when-ai-fails\/. (Accessed on 09\/16\/2020)."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3026044"},{"key":"e_1_3_2_1_26_1","unstructured":"Kuang Chen Joseph\u00a0M Hellerstein and Tapan\u00a0S Parikh. 2011. Data in the First Mile.. In CIDR. Citeseer 203\u2013206.  Kuang Chen Joseph\u00a0M Hellerstein and Tapan\u00a0S Parikh. 2011. Data in the First Mile.. In CIDR. Citeseer 203\u2013206."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE.2013.6544847"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"crossref","unstructured":"Josh Cowls Thomas King Mariarosaria Taddeo and Luciano Floridi. 2019. Designing AI for social good: Seven essential factors. Available at SSRN 3388669(2019).  Josh Cowls Thomas King Mariarosaria Taddeo and Luciano Floridi. 2019. Designing AI for social good: Seven essential factors. Available at SSRN 3388669(2019).","DOI":"10.2139\/ssrn.3388669"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/157710.157715"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3148148"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3210548"},{"key":"e_1_3_2_1_32_1","unstructured":"Alan Dix Alan\u00a0John Dix Janet Finlay Gregory\u00a0D Abowd and Russell Beale. 2003. Human-computer interaction. Pearson Education.  Alan Dix Alan\u00a0John Dix Janet Finlay Gregory\u00a0D Abowd and Russell Beale. 2003. Human-computer interaction. Pearson Education."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1177\/2053951719858751"},{"key":"e_1_3_2_1_34_1","volume-title":"Heteromation and its (dis) contents: The invisible division of labor between humans and machines. First Monday","author":"Ekbia Hamid","year":"2014","unstructured":"Hamid Ekbia and Bonnie Nardi . 2014. Heteromation and its (dis) contents: The invisible division of labor between humans and machines. First Monday ( 2014 ). Hamid Ekbia and Bonnie Nardi. 2014. Heteromation and its (dis) contents: The invisible division of labor between humans and machines. First Monday (2014)."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025837"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/1064978.1065046"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11023-018-9482-5"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/585058.585065"},{"key":"e_1_3_2_1_39_1","unstructured":"Martin Fowler. 2019. TechnicalDebt. https:\/\/martinfowler.com\/bliki\/TechnicalDebt.html. (Accessed on 09\/16\/2020).  Martin Fowler. 2019. TechnicalDebt. https:\/\/martinfowler.com\/bliki\/TechnicalDebt.html. (Accessed on 09\/16\/2020)."},{"key":"e_1_3_2_1_40_1","unstructured":"Timnit Gebru Jamie Morgenstern Briana Vecchione Jennifer\u00a0Wortman Vaughan Hanna Wallach Hal Daum\u00e9\u00a0III and Kate Crawford. 2018. Datasheets for datasets. (2018). arXiv:1803.09010  Timnit Gebru Jamie Morgenstern Briana Vecchione Jennifer\u00a0Wortman Vaughan Hanna Wallach Hal Daum\u00e9\u00a0III and Kate Crawford. 2018. Datasheets for datasets. (2018). arXiv:1803.09010"},{"key":"e_1_3_2_1_41_1","volume-title":"Raw data is an oxymoron","author":"Gitelman Lisa","unstructured":"Lisa Gitelman . 2013. Raw data is an oxymoron . MIT press . Lisa Gitelman. 2013. Raw data is an oxymoron. MIT press."},{"key":"e_1_3_2_1_42_1","volume-title":"Deep learning. Vol.\u00a01","author":"Goodfellow Ian","unstructured":"Ian Goodfellow , Yoshua Bengio , Aaron Courville , and Yoshua Bengio . 2016. Deep learning. Vol.\u00a01 . MIT press Cambridge . Ian Goodfellow, Yoshua Bengio, Aaron Courville, and Yoshua Bengio. 2016. Deep learning. Vol.\u00a01. MIT press Cambridge."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/1066157.1066252"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIS.2009.36"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA.2018.00059"},{"key":"e_1_3_2_1_46_1","unstructured":"Benjamin Heinzerling. 2020. NLP\u2019s Clever Hans Moment has Arrived. https:\/\/thegradient.pub\/nlps-clever-hans-moment-has-arrived\/  Benjamin Heinzerling. 2020. NLP\u2019s Clever Hans Moment has Arrived. https:\/\/thegradient.pub\/nlps-clever-hans-moment-has-arrived\/"},{"key":"e_1_3_2_1_47_1","volume-title":"Tech folk: \u2019Move fast and break things","author":"Hiatt Keith","year":"2017","unstructured":"Keith Hiatt , Michael Kleinman , and Mark Latonero . [n.d.]. Tech folk: \u2019Move fast and break things \u2019 doesn\u2019t work when lives are at stake | The Guardian. https:\/\/www.theguardian.com\/global-development-professionals-network\/ 2017 \/feb\/02\/technology-human-rights. (Accessed on 08\/25\/2020). Keith Hiatt, Michael Kleinman, and Mark Latonero. [n.d.]. Tech folk: \u2019Move fast and break things\u2019 doesn\u2019t work when lives are at stake | The Guardian. https:\/\/www.theguardian.com\/global-development-professionals-network\/2017\/feb\/02\/technology-human-rights. (Accessed on 08\/25\/2020)."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2016.7739680"},{"key":"e_1_3_2_1_49_1","volume-title":"Fifteenth National Conference on Artificial Intelligence (AAAI-98)","author":"Hirschberg J","year":"1998","unstructured":"J Hirschberg . 1998 . Every time I fire a linguist, my performance goes up, and other myths of the statistical natural language processing revolution. Invited talk . In Fifteenth National Conference on Artificial Intelligence (AAAI-98) . J Hirschberg. 1998. Every time I fire a linguist, my performance goes up, and other myths of the statistical natural language processing revolution. Invited talk. In Fifteenth National Conference on Artificial Intelligence (AAAI-98)."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/2736277.2741102"},{"key":"e_1_3_2_1_51_1","volume-title":"A survey of outlier detection methodologies. Artificial intelligence review 22, 2","author":"Hodge Victoria","year":"2004","unstructured":"Victoria Hodge and Jim Austin . 2004. A survey of outlier detection methodologies. Artificial intelligence review 22, 2 ( 2004 ), 85\u2013126. Victoria Hodge and Jim Austin. 2004. A survey of outlier detection methodologies. Artificial intelligence review 22, 2 (2004), 85\u2013126."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376177"},{"key":"e_1_3_2_1_53_1","volume-title":"Alex Hanna, Emily Denton, Christina Greer, Oddur Kjartansson, Parker Barnes, and Margaret Mitchell.","author":"Hutchinson Ben","year":"2020","unstructured":"Ben Hutchinson , Andrew Smart , Alex Hanna, Emily Denton, Christina Greer, Oddur Kjartansson, Parker Barnes, and Margaret Mitchell. 2020 . Towards Accountability for Machine Learning Datasets: Practices from Software Engineering and Infrastructure . (2020). arXiv:2010.13561 Ben Hutchinson, Andrew Smart, Alex Hanna, Emily Denton, Christina Greer, Oddur Kjartansson, Parker Barnes, and Margaret Mitchell. 2020. Towards Accountability for Machine Learning Datasets: Practices from Software Engineering and Infrastructure. (2020). arXiv:2010.13561"},{"key":"e_1_3_2_1_54_1","volume-title":"NIPS MLSys Workshop.","author":"Hynes Nick","year":"2017","unstructured":"Nick Hynes , D Sculley , and Michael Terry . 2017 . The data linter: Lightweight, automated sanity checking for ml data sets . In NIPS MLSys Workshop. Nick Hynes, D Sculley, and Michael Terry. 2017. The data linter: Lightweight, automated sanity checking for ml data sets. In NIPS MLSys Workshop."},{"key":"e_1_3_2_1_55_1","volume-title":"Increasing value and reducing waste in research design, conduct, and analysis. The Lancet 383, 9912","author":"Ioannidis PA","year":"2014","unstructured":"John\u00a0 PA Ioannidis , Sander Greenland , Mark\u00a0 A Hlatky , Muin\u00a0 J Khoury , Malcolm\u00a0 R Macleod , David Moher , Kenneth\u00a0 F Schulz , and Robert Tibshirani . 2014. Increasing value and reducing waste in research design, conduct, and analysis. The Lancet 383, 9912 ( 2014 ), 166\u2013175. John\u00a0PA Ioannidis, Sander Greenland, Mark\u00a0A Hlatky, Muin\u00a0J Khoury, Malcolm\u00a0R Macleod, David Moher, Kenneth\u00a0F Schulz, and Robert Tibshirani. 2014. Increasing value and reducing waste in research design, conduct, and analysis. The Lancet 383, 9912 (2014), 166\u2013175."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1177\/1461444813511926"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/2470654.2470742"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274345"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"crossref","unstructured":"Ayush Jain Akash\u00a0Das Sarma Aditya Parameswaran and Jennifer Widom. 2017. Understanding workers developing effective tasks and enhancing marketplace dynamics: a study of a large crowdsourcing marketplace. (2017). arXiv:1701.06207  Ayush Jain Akash\u00a0Das Sarma Aditya Parameswaran and Jennifer Widom. 2017. Understanding workers developing effective tasks and enhancing marketplace dynamics: a study of a large crowdsourcing marketplace. (2017). arXiv:1701.06207","DOI":"10.14778\/3067421.3067431"},{"key":"e_1_3_2_1_60_1","unstructured":"Kaggle. 2019. 2019 Kaggle ML & DS Survey. https:\/\/www.kaggle.com\/c\/kaggle-survey-2019. (Accessed on 08\/27\/2020).  Kaggle. 2019. 2019 Kaggle ML & DS Survey. https:\/\/www.kaggle.com\/c\/kaggle-survey-2019. (Accessed on 08\/27\/2020)."},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979444"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2012.219"},{"key":"e_1_3_2_1_63_1","volume-title":"Reappraising the utility of Google Flu Trends. PLoS computational biology 15, 8","author":"Kandula Sasikiran","year":"2019","unstructured":"Sasikiran Kandula and Jeffrey Shaman . 2019. Reappraising the utility of Google Flu Trends. PLoS computational biology 15, 8 ( 2019 ), e1007258. Sasikiran Kandula and Jeffrey Shaman. 2019. Reappraising the utility of Google Flu Trends. PLoS computational biology 15, 8 (2019), e1007258."},{"key":"e_1_3_2_1_64_1","unstructured":"Hannah Kerner. [n.d.]. Too many AI researchers think real-world problems are not relevant | MIT Technology Review. https:\/\/www.technologyreview.com\/2020\/08\/18\/1007196\/ai-research-machine-learning-applications-problems-opinion\/. (Accessed on 08\/18\/2020).  Hannah Kerner. [n.d.]. Too many AI researchers think real-world problems are not relevant | MIT Technology Review. https:\/\/www.technologyreview.com\/2020\/08\/18\/1007196\/ai-research-machine-learning-applications-problems-opinion\/. (Accessed on 08\/18\/2020)."},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025626"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2017.2754374"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/2501105.2501106"},{"key":"e_1_3_2_1_68_1","unstructured":"Laura Koesten Kathleen Gregory Paul Groth and Elena Simperl. 2019. Talking datasets: Understanding data sensemaking behaviours. (2019). arXiv:1911.09041  Laura Koesten Kathleen Gregory Paul Groth and Elena Simperl. 2019. Talking datasets: Understanding data sensemaking behaviours. (2019). arXiv:1911.09041"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300330"},{"key":"e_1_3_2_1_70_1","volume-title":"Boostclean: Automated error detection and repair for machine learning.","author":"Krishnan Sanjay","year":"2017","unstructured":"Sanjay Krishnan , Michael\u00a0 J Franklin , Ken Goldberg , and Eugene Wu . 2017 . Boostclean: Automated error detection and repair for machine learning. (2017). arXiv:1711.01299 Sanjay Krishnan, Michael\u00a0J Franklin, Ken Goldberg, and Eugene Wu. 2017. Boostclean: Automated error detection and repair for machine learning. (2017). arXiv:1711.01299"},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939502.2939511"},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.14778\/2994509.2994514"},{"key":"e_1_3_2_1_73_1","unstructured":"David Lazer and Ryan Kennedy. 2015. What We Can Learn From the Epic Failure of Google Flu Trends | WIRED. https:\/\/www.wired.com\/2015\/10\/can-learn-epic-failure-google-flu-trends\/. (Accessed on 08\/27\/2020).  David Lazer and Ryan Kennedy. 2015. What We Can Learn From the Epic Failure of Google Flu Trends | WIRED. https:\/\/www.wired.com\/2015\/10\/can-learn-epic-failure-google-flu-trends\/. (Accessed on 08\/27\/2020)."},{"key":"e_1_3_2_1_74_1","unstructured":"Zachary\u00a0C Lipton and Jacob Steinhardt. 2018. Troubling trends in machine learning scholarship. (2018). arXiv:1807.03341  Zachary\u00a0C Lipton and Jacob Steinhardt. 2018. Troubling trends in machine learning scholarship. (2018). arXiv:1807.03341"},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"crossref","unstructured":"Maria Littmann Katharina Selig Liel Cohen-Lavi Yotam Frank Peter H\u00f6nigschmid Evans Kataka Anja M\u00f6sch Kun Qian Avihai Ron Sebastian Schmid 2020. Validity of machine learning in biology and medicine increased through collaborations across fields of expertise. Nature Machine Intelligence(2020) 1\u20137.  Maria Littmann Katharina Selig Liel Cohen-Lavi Yotam Frank Peter H\u00f6nigschmid Evans Kataka Anja M\u00f6sch Kun Qian Avihai Ron Sebastian Schmid 2020. Validity of machine learning in biology and medicine increased through collaborations across fields of expertise. Nature Machine Intelligence(2020) 1\u20137.","DOI":"10.1038\/s42256-019-0139-8"},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1145\/3329486.3329489"},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"crossref","unstructured":"Yaoli Mao Dakuo Wang Michael Muller Kush\u00a0R Varshney Ioana Baldini Casey Dugan and Aleksandra Mojsilovi\u0107. 2019. How Data Scientists Work Together With Domain Experts in Scientific Collaborations: To Find The Right Answer Or To Ask The Right Question?Proceedings of the ACM on Human-Computer Interaction 3 GROUP(2019) 1\u201323.  Yaoli Mao Dakuo Wang Michael Muller Kush\u00a0R Varshney Ioana Baldini Casey Dugan and Aleksandra Mojsilovi\u0107. 2019. How Data Scientists Work Together With Domain Experts in Scientific Collaborations: To Find The Right Answer Or To Ask The Right Question?Proceedings of the ACM on Human-Computer Interaction 3 GROUP(2019) 1\u201323.","DOI":"10.1145\/3361118"},{"key":"e_1_3_2_1_78_1","unstructured":"Gary Marcus. 2018. Deep learning: A critical appraisal. (2018). arXiv:1801.00631  Gary Marcus. 2018. Deep learning: A critical appraisal. (2018). arXiv:1801.00631"},{"key":"e_1_3_2_1_79_1","doi-asserted-by":"publisher","DOI":"10.1145\/2531602.2531663"},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359174"},{"key":"e_1_3_2_1_81_1","unstructured":"Ninareh Mehrabi Fred Morstatter Nripsuta Saxena Kristina Lerman and Aram Galstyan. 2019. A survey on bias and fairness in machine learning. (2019). arXiv:1908.09635  Ninareh Mehrabi Fred Morstatter Nripsuta Saxena Kristina Lerman and Aram Galstyan. 2019. A survey on bias and fairness in machine learning. (2019). arXiv:1908.09635"},{"key":"e_1_3_2_1_82_1","volume-title":"The multimodal brain tumor image segmentation benchmark (BRATS)","author":"Menze H","year":"2014","unstructured":"Bjoern\u00a0 H Menze , Andras Jakab , Stefan Bauer , Jayashree Kalpathy-Cramer , Keyvan Farahani , Justin Kirby , Yuliya Burren , Nicole Porz , Johannes Slotboom , Roland Wiest , 2014. The multimodal brain tumor image segmentation benchmark (BRATS) . IEEE transactions on medical imaging 34, 10 ( 2014 ), 1993\u20132024. Bjoern\u00a0H Menze, Andras Jakab, Stefan Bauer, Jayashree Kalpathy-Cramer, Keyvan Farahani, Justin Kirby, Yuliya Burren, Nicole Porz, Johannes Slotboom, Roland Wiest, 2014. The multimodal brain tumor image segmentation benchmark (BRATS). IEEE transactions on medical imaging 34, 10 (2014), 1993\u20132024."},{"key":"e_1_3_2_1_83_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2019.2954841"},{"key":"e_1_3_2_1_84_1","unstructured":"Hannah Miller and Richard Stirling. 2019. Government AI Readiness Index 2019 \u2014 Oxford Insights \u2014 Oxford Insights. https:\/\/www.oxfordinsights.com\/ai-readiness2019. (Accessed on 09\/14\/2020).  Hannah Miller and Richard Stirling. 2019. Government AI Readiness Index 2019 \u2014 Oxford Insights \u2014 Oxford Insights. https:\/\/www.oxfordinsights.com\/ai-readiness2019. (Accessed on 09\/14\/2020)."},{"key":"e_1_3_2_1_85_1","volume-title":"Who does the work of data?Interactions 27, 3","author":"M\u00f8ller Naja\u00a0Holten","year":"2020","unstructured":"Naja\u00a0Holten M\u00f8ller , Claus Bossen , Kathleen\u00a0 H Pine , Trine\u00a0Rask Nielsen , and Gina Neff . 2020. Who does the work of data?Interactions 27, 3 ( 2020 ), 52\u201355. Naja\u00a0Holten M\u00f8ller, Claus Bossen, Kathleen\u00a0H Pine, Trine\u00a0Rask Nielsen, and Gina Neff. 2020. Who does the work of data?Interactions 27, 3 (2020), 52\u201355."},{"key":"e_1_3_2_1_86_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300356"},{"key":"e_1_3_2_1_87_1","unstructured":"Tadhg Nagle C.\u00a0Thomas Redman and David Sammon. 2017. Only 3% of Companies\u2019 Data Meets Basic Quality Standards. https:\/\/hbr.org\/2017\/09\/only-3-of-companies-data-meets-basic-quality-standards. (Accessed on 08\/27\/2020).  Tadhg Nagle C.\u00a0Thomas Redman and David Sammon. 2017. Only 3% of Companies\u2019 Data Meets Basic Quality Standards. https:\/\/hbr.org\/2017\/09\/only-3-of-companies-data-meets-basic-quality-standards. (Accessed on 08\/27\/2020)."},{"key":"e_1_3_2_1_88_1","volume-title":"Algorithms of oppression: How search engines reinforce racism","author":"Noble Safiya\u00a0Umoja","unstructured":"Safiya\u00a0Umoja Noble . 2018. Algorithms of oppression: How search engines reinforce racism . NYU Press . Safiya\u00a0Umoja Noble. 2018. Algorithms of oppression: How search engines reinforce racism. NYU Press."},{"key":"e_1_3_2_1_89_1","volume-title":"Purposeful sampling for qualitative data collection and analysis in mixed method implementation research. Administration and policy in mental health and mental health services research 42, 5","author":"Palinkas A","year":"2015","unstructured":"Lawrence\u00a0 A Palinkas , Sarah\u00a0 M Horwitz , Carla\u00a0 A Green , Jennifer\u00a0 P Wisdom , Naihua Duan , and Kimberly Hoagwood . 2015. Purposeful sampling for qualitative data collection and analysis in mixed method implementation research. Administration and policy in mental health and mental health services research 42, 5 ( 2015 ), 533\u2013544. Lawrence\u00a0A Palinkas, Sarah\u00a0M Horwitz, Carla\u00a0A Green, Jennifer\u00a0P Wisdom, Naihua Duan, and Kimberly Hoagwood. 2015. Purposeful sampling for qualitative data collection and analysis in mixed method implementation research. Administration and policy in mental health and mental health services research 42, 5 (2015), 533\u2013544."},{"key":"e_1_3_2_1_90_1","doi-asserted-by":"publisher","DOI":"10.1145\/3184558.3191551"},{"key":"e_1_3_2_1_91_1","volume-title":"Evaluating Evaluation of AI Systems (Meta-Eval","author":"Paritosh Praveen","year":"2020","unstructured":"Praveen Paritosh , Kurt Bollacker , Maria Stone , Lora Aroyo , and Sarah Luger . 2020. Evaluating Evaluation of AI Systems (Meta-Eval 2020 ). http:\/\/eval.how\/aaai-2020\/. ( Accessed on 09\/16\/2020). Praveen Paritosh, Kurt Bollacker, Maria Stone, Lora Aroyo, and Sarah Luger. 2020. Evaluating Evaluation of AI Systems (Meta-Eval 2020). http:\/\/eval.how\/aaai-2020\/. (Accessed on 09\/16\/2020)."},{"key":"e_1_3_2_1_92_1","volume-title":"First workshop on Data Excellence (DEW 2020","author":"Paritosh Praveen","year":"2020","unstructured":"Praveen Paritosh , Matt Lease , Mike Schaekermann , and Lora Aroyo . 2020 . First workshop on Data Excellence (DEW 2020 ). http:\/\/eval.how\/dew2020\/. (Accessed on 09\/16\/2020). Praveen Paritosh, Matt Lease, Mike Schaekermann, and Lora Aroyo. 2020. First workshop on Data Excellence (DEW 2020). http:\/\/eval.how\/dew2020\/. (Accessed on 09\/16\/2020)."},{"key":"e_1_3_2_1_93_1","doi-asserted-by":"publisher","DOI":"10.1145\/2998181.2998331"},{"key":"e_1_3_2_1_94_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274405"},{"key":"e_1_3_2_1_95_1","doi-asserted-by":"publisher","DOI":"10.1177\/2053951720939605"},{"key":"e_1_3_2_1_96_1","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357160"},{"key":"e_1_3_2_1_97_1","volume-title":"The secret life of pronouns. New Scientist 211, 2828","author":"Pennebaker W","year":"2011","unstructured":"James\u00a0 W Pennebaker . 2011. The secret life of pronouns. New Scientist 211, 2828 ( 2011 ), 42\u201345. James\u00a0W Pennebaker. 2011. The secret life of pronouns. New Scientist 211, 2828 (2011), 42\u201345."},{"key":"e_1_3_2_1_98_1","doi-asserted-by":"publisher","DOI":"10.1145\/3314344.3332496"},{"key":"e_1_3_2_1_99_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702298"},{"key":"e_1_3_2_1_100_1","doi-asserted-by":"publisher","DOI":"10.1145\/3035918.3054782"},{"key":"e_1_3_2_1_101_1","doi-asserted-by":"publisher","DOI":"10.1145\/3299887.3299891"},{"key":"e_1_3_2_1_102_1","unstructured":"Vijayshankar Raman and Joseph\u00a0M Hellerstein. 2001. Potter\u2019s wheel: An interactive data cleaning system. In VLDB Vol.\u00a01. 381\u2013390.  Vijayshankar Raman and Joseph\u00a0M Hellerstein. 2001. Potter\u2019s wheel: An interactive data cleaning system. In VLDB Vol.\u00a01. 381\u2013390."},{"key":"e_1_3_2_1_103_1","unstructured":"Thomas\u00a0C. Redman. 2018. If Your Data Is Bad Your Machine Learning Tools Are Useless. https:\/\/hbr.org\/2018\/04\/if-your-data-is-bad-your-machine-learning-tools-are-useless  Thomas\u00a0C. Redman. 2018. If Your Data Is Bad Your Machine Learning Tools Are Useless. https:\/\/hbr.org\/2018\/04\/if-your-data-is-bad-your-machine-learning-tools-are-useless"},{"key":"e_1_3_2_1_104_1","first-page":"15","article-title":"Dirty data, bad predictions: How civil rights violations impact police data, predictive policing systems, and justice","volume":"94","author":"Richardson Rashida","year":"2019","unstructured":"Rashida Richardson , Jason\u00a0 M Schultz , and Kate Crawford . 2019 . Dirty data, bad predictions: How civil rights violations impact police data, predictive policing systems, and justice . NYUL Rev. Online 94 (2019), 15 . Rashida Richardson, Jason\u00a0M Schultz, and Kate Crawford. 2019. Dirty data, bad predictions: How civil rights violations impact police data, predictive policing systems, and justice. NYUL Rev. Online 94(2019), 15.","journal-title":"NYUL Rev. Online"},{"key":"e_1_3_2_1_105_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341164"},{"key":"e_1_3_2_1_106_1","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2017.8258190"},{"key":"e_1_3_2_1_107_1","doi-asserted-by":"crossref","unstructured":"Nithya Sambasivan Erin Arnesen Ben Hutchinson Tulsee Doshi and Vinodkumar Prabhakaran. 2021. Re-imagining Algorithmic Fairness in India and Beyond. In ACM FaccT.  Nithya Sambasivan Erin Arnesen Ben Hutchinson Tulsee Doshi and Vinodkumar Prabhakaran. 2021. Re-imagining Algorithmic Fairness in India and Beyond. In ACM FaccT.","DOI":"10.1145\/3442188.3445896"},{"key":"e_1_3_2_1_108_1","volume-title":"Women in South Asia. In Fourteenth Symposium on Usable Privacy and Security ({SOUPS}","author":"Sambasivan Nithya","year":"2018","unstructured":"Nithya Sambasivan , Garen Checkley , Amna Batool , Nova Ahmed , David Nemer , Laura\u00a0Sanely Gayt\u00e1n-Lugo , Tara Matthews , Sunny Consolvo , and Elizabeth Churchill . 2018 . \u201d Privacy is not for me, it\u2019s for those rich women\u201d: Performative Privacy Practices on Mobile Phones by Women in South Asia. In Fourteenth Symposium on Usable Privacy and Security ({SOUPS} 2018). 127\u2013142. Nithya Sambasivan, Garen Checkley, Amna Batool, Nova Ahmed, David Nemer, Laura\u00a0Sanely Gayt\u00e1n-Lugo, Tara Matthews, Sunny Consolvo, and Elizabeth Churchill. 2018. \u201d Privacy is not for me, it\u2019s for those rich women\u201d: Performative Privacy Practices on Mobile Phones by Women in South Asia. In Fourteenth Symposium on Usable Privacy and Security ({SOUPS} 2018). 127\u2013142."},{"key":"e_1_3_2_1_109_1","volume-title":"Toward responsible AI for the next billion users. interactions 26, 1","author":"Sambasivan Nithya","year":"2018","unstructured":"Nithya Sambasivan and Jess Holbrook . 2018. Toward responsible AI for the next billion users. interactions 26, 1 ( 2018 ), 68\u201371. Nithya Sambasivan and Jess Holbrook. 2018. Toward responsible AI for the next billion users. interactions 26, 1 (2018), 68\u201371."},{"key":"e_1_3_2_1_110_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359246"},{"key":"e_1_3_2_1_111_1","unstructured":"David Sculley Gary Holt Daniel Golovin Eugene Davydov Todd Phillips Dietmar Ebner Vinay Chaudhary Michael Young Jean-Francois Crespo and Dan Dennison. 2015. Hidden technical debt in machine learning systems. In Advances in neural information processing systems. 2503\u20132511.  David Sculley Gary Holt Daniel Golovin Eugene Davydov Todd Phillips Dietmar Ebner Vinay Chaudhary Michael Young Jean-Francois Crespo and Dan Dennison. 2015. Hidden technical debt in machine learning systems. In Advances in neural information processing systems. 2503\u20132511."},{"key":"e_1_3_2_1_112_1","unstructured":"David Sculley Jasper Snoek Alex Wiltschko and Ali Rahimi. 2018. Winner\u2019s curse? On pace progress and empirical rigor. (2018).  David Sculley Jasper Snoek Alex Wiltschko and Ali Rahimi. 2018. Winner\u2019s curse? On pace progress and empirical rigor. (2018)."},{"key":"e_1_3_2_1_113_1","unstructured":"Zheyuan\u00a0Ryan Shi Claire Wang and Fei Fang. 2020. Artificial Intelligence for Social Good: A Survey. arxiv:2001.01818\u00a0[cs.CY]  Zheyuan\u00a0Ryan Shi Claire Wang and Fei Fang. 2020. Artificial Intelligence for Social Good: A Survey. arxiv:2001.01818\u00a0[cs.CY]"},{"key":"e_1_3_2_1_114_1","unstructured":"David Soergel Adam Saunders and Andrew McCallum. 2013. Open Scholarship and Peer Review: a Time for Experimentation. (2013).  David Soergel Adam Saunders and Andrew McCallum. 2013. Open Scholarship and Peer Review: a Time for Experimentation. (2013)."},{"key":"e_1_3_2_1_115_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSPEC.2019.8678513"},{"key":"e_1_3_2_1_116_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.giq.2018.11.002"},{"key":"e_1_3_2_1_117_1","unstructured":"Astra Taylor. 2018. The Automation Charade. https:\/\/logicmag.io\/failure\/the-automation-charade\/.  Astra Taylor. 2018. The Automation Charade. https:\/\/logicmag.io\/failure\/the-automation-charade\/."},{"key":"e_1_3_2_1_118_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702558"},{"key":"e_1_3_2_1_119_1","volume-title":"A general inductive approach for analyzing qualitative evaluation data. American journal of evaluation 27, 2","author":"Thomas R","year":"2006","unstructured":"David\u00a0 R Thomas . 2006. A general inductive approach for analyzing qualitative evaluation data. American journal of evaluation 27, 2 ( 2006 ), 237\u2013246. David\u00a0R Thomas. 2006. A general inductive approach for analyzing qualitative evaluation data. American journal of evaluation 27, 2 (2006), 237\u2013246."},{"key":"e_1_3_2_1_120_1","unstructured":"Rachel Thomas and David Uminsky. 2020. The Problem with Metrics is a Fundamental Problem for AI. (2020). arXiv:2002.08512  Rachel Thomas and David Uminsky. 2020. The Problem with Metrics is a Fundamental Problem for AI. (2020). arXiv:2002.08512"},{"key":"e_1_3_2_1_121_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-020-15871-z"},{"key":"e_1_3_2_1_122_1","doi-asserted-by":"publisher","DOI":"10.5555\/3122009.3242050"},{"key":"e_1_3_2_1_123_1","doi-asserted-by":"publisher","DOI":"10.1145\/1958824.1958906"},{"key":"e_1_3_2_1_124_1","unstructured":"Bret Victor. 2013. Media for Thinking the Unthinkable. http:\/\/worrydream.com\/MediaForThinkingTheUnthinkable\/. (Accessed on 09\/15\/2020).  Bret Victor. 2013. Media for Thinking the Unthinkable. http:\/\/worrydream.com\/MediaForThinkingTheUnthinkable\/. (Accessed on 09\/15\/2020)."},{"key":"e_1_3_2_1_125_1","unstructured":"Kiri Wagstaff. 2012. Machine learning that matters. (2012). arXiv:1206.4656  Kiri Wagstaff. 2012. Machine learning that matters. (2012). arXiv:1206.4656"},{"key":"e_1_3_2_1_126_1","volume-title":"Discriminating systems: Gender, race and power in AI","author":"West Sarah\u00a0Myers","year":"2019","unstructured":"Sarah\u00a0Myers West , Meredith Whittaker , and Kate Crawford . 2019. Discriminating systems: Gender, race and power in AI . AI Now Institute ( 2019 ), 1\u201333. Sarah\u00a0Myers West, Meredith Whittaker, and Kate Crawford. 2019. Discriminating systems: Gender, race and power in AI. AI Now Institute (2019), 1\u201333."},{"key":"e_1_3_2_1_127_1","doi-asserted-by":"publisher","DOI":"10.1145\/3392826"},{"key":"e_1_3_2_1_128_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10462-016-9491-9"},{"key":"e_1_3_2_1_129_1","volume-title":"Machine learning testing: Survey, landscapes and horizons","author":"Zhang M","year":"2020","unstructured":"Jie\u00a0 M Zhang , Mark Harman , Lei Ma , and Yang Liu . 2020. Machine learning testing: Survey, landscapes and horizons . IEEE Transactions on Software Engineering( 2020 ). Jie\u00a0M Zhang, Mark Harman, Lei Ma, and Yang Liu. 2020. Machine learning testing: Survey, landscapes and horizons. IEEE Transactions on Software Engineering(2020)."}],"event":{"name":"CHI '21: CHI Conference on Human Factors in Computing Systems","location":"Yokohama Japan","acronym":"CHI '21","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3411764.3445518","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3411764.3445518","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T21:28:48Z","timestamp":1750195728000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3411764.3445518"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,5,6]]},"references-count":129,"alternative-id":["10.1145\/3411764.3445518","10.1145\/3411764"],"URL":"https:\/\/doi.org\/10.1145\/3411764.3445518","relation":{},"subject":[],"published":{"date-parts":[[2021,5,6]]},"assertion":[{"value":"2021-05-07","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}