{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T20:54:02Z","timestamp":1776113642903,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":31,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T00:00:00Z","timestamp":1655683200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,6,21]]},"DOI":"10.1145\/3531146.3533122","type":"proceedings-article","created":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T14:27:10Z","timestamp":1655735230000},"page":"587-597","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":26,"title":["REAL ML: Recognizing, Exploring, and Articulating Limitations of Machine Learning Research"],"prefix":"10.1145","author":[{"given":"Jessie J.","family":"Smith","sequence":"first","affiliation":[{"name":"University of Colorado, Boulder, USA"}]},{"given":"Saleema","family":"Amershi","sequence":"additional","affiliation":[{"name":"Microsoft Research, USA"}]},{"given":"Solon","family":"Barocas","sequence":"additional","affiliation":[{"name":"Microsoft Research, USA"}]},{"given":"Hanna","family":"Wallach","sequence":"additional","affiliation":[{"name":"Microsoft Research, USA"}]},{"given":"Jennifer","family":"Wortman Vaughan","sequence":"additional","affiliation":[{"name":"Microsoft Research, USA"}]}],"member":"320","published-online":{"date-parts":[[2022,6,20]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"Alina Beygelzimer Yann Dauphin Percy Liang and Jennifer\u00a0Wortman Vaughan. 2021. Introducing the NeurIPS 2021 Paper Checklist. (2021). https:\/\/neuripsconf.medium.com\/introducing-the-neurips-2021-paper-checklist-3220d6df500b Medium Article."},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"crossref","unstructured":"Abeba Birhane Pratyusha Kalluri Dallas Card William Agnew Ravit Dotan and Michelle Bao. 2021. The values encoded in machine learning research. arXiv preprint arXiv:2106.15590(2021).","DOI":"10.1145\/3531146.3533083"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1710755115"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1177\/0149206312455245"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.econlet.2018.03.036"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1257\/jel.20171350"},{"key":"e_1_3_2_2_7_1","volume-title":"Encyclopedia of critical psychology","author":"Clarke Victoria","year":"1947","unstructured":"Victoria Clarke and Virginia Braun. 2014. Thematic analysis. In Encyclopedia of critical psychology. Springer, 1947\u20131952."},{"key":"e_1_3_2_2_8_1","volume-title":"Data Feminism","author":"D\u2019ignazio Catherine","unstructured":"Catherine D\u2019ignazio and Lauren\u00a0F Klein. 2020. Data Feminism. MIT press."},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3458723"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/1964897.1964899"},{"key":"e_1_3_2_2_11_1","volume-title":"Structured abstracts. Narrative review. Acta Cir\u00fargica Brasileira21","author":"Guimar\u00e3es Carlos\u00a0Alberto","year":"2006","unstructured":"Carlos\u00a0Alberto Guimar\u00e3es. 2006. Structured abstracts. Narrative review. Acta Cir\u00fargica Brasileira21 (2006), 263\u2013268."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jclinepi.2006.09.011"},{"key":"e_1_3_2_2_13_1","unstructured":"JAMA. 2021. Instructions for Authors. (2021). https:\/\/jamanetwork.com\/journals\/jama\/pages\/instructions-for-authors Publishing Instructions."},{"key":"e_1_3_2_2_14_1","unstructured":"JNeuroSci. 2021. Information for Authors. (2021). https:\/\/www.jneurosci.org\/content\/information-authors Publishing Instructions."},{"key":"e_1_3_2_2_15_1","volume-title":"Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2).","author":"Liao Thomas","year":"2021","unstructured":"Thomas Liao, Rohan Taori, Inioluwa\u00a0Deborah Raji, and Ludwig Schmidt. 2021. Are We Learning Yet? A Meta Review of Evaluation Failures Across Machine Learning. In Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)."},{"key":"e_1_3_2_2_16_1","volume-title":"Getting Started with NeurIPS","author":"Lin Hsuan-Tien","year":"2020","unstructured":"Hsuan-Tien Lin, Maria-Florina Balcan, Raia Hadsell, and Marc\u2019Aurelio Ranzato. 2020. Getting Started with NeurIPS 2020. (2020). https:\/\/neuripsconf.medium.com\/getting-started-with-neurips-2020-e350f9b39c28 Medium Article."},{"key":"e_1_3_2_2_17_1","volume-title":"Story, Not Study: 30 Brief Lessons to Inspire Health Researchers as Writers","author":"Lingard Lorelei","unstructured":"Lorelei Lingard and Christopher Watling. 2021. The art of limitations. In Story, Not Study: 30 Brief Lessons to Inspire Health Researchers as Writers. Springer, 53\u201359."},{"key":"e_1_3_2_2_18_1","volume-title":"Toward a more perfect psychology: Improving trust, accuracy, and transparency in research","author":"Makel C","unstructured":"Matthew\u00a0C Makel and Jonathan\u00a0A Plucker. 2017. Toward a more perfect psychology: Improving trust, accuracy, and transparency in research.American Psychological Association."},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287596"},{"key":"e_1_3_2_2_20_1","volume-title":"Reflection-in-action and reflection-on-action. Current issues in education 9, 1","author":"Munby Hugh","year":"1989","unstructured":"Hugh Munby. 1989. Reflection-in-action and reflection-on-action. Current issues in education 9, 1 (1989), 31\u201342."},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3461702.3462608"},{"key":"e_1_3_2_2_22_1","unstructured":"Arvind Narayanan. 2019. How to recognize AI snake oil. Arthur Miller Lecture on Science and Ethics(2019)."},{"key":"e_1_3_2_2_23_1","volume-title":"Managing the Risks of AI Research: Six Recommendations for Responsible Publication. white paper","author":"Partnership","year":"2021","unstructured":"Partnership on AI. 2021. Managing the Risks of AI Research: Six Recommendations for Responsible Publication. white paper (2021)."},{"key":"e_1_3_2_2_24_1","volume-title":"Discussing study limitations in reports of biomedical studies-the need for more transparency. Health and quality of life outcomes 10, 1","author":"Puhan A","year":"2012","unstructured":"Milo\u00a0A Puhan, Elie\u00a0A Akl, Dianne Bryant, Feng Xie, Giovanni Apolone, and Gerben ter Riet. 2012. Discussing study limitations in reports of biomedical studies-the need for more transparency. Health and quality of life outcomes 10, 1 (2012), 1\u20134."},{"key":"e_1_3_2_2_25_1","volume-title":"Limited by our limitations. Perspectives on medical education 8, 4","author":"Ross T","year":"2019","unstructured":"Paula\u00a0T Ross and Nikki L\u00a0Bibler Zaidi. 2019. Limited by our limitations. Perspectives on medical education 8, 4 (2019), 261\u2013264."},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1080\/10464883.1984.10758345"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287598"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445971"},{"key":"e_1_3_2_2_29_1","unstructured":"Kate Sim Andrew Brown and Amelia Hassoun. 2021. Thinking Through and Writing About Research Ethics Beyond \u201cBroader Impact\u201d. arXiv preprint arXiv:2104.08205(2021)."},{"key":"e_1_3_2_2_30_1","volume-title":"Proceedings of the 29th International Coference on International Conference on Machine Learning. 1851\u20131856","author":"Wagstaff L","year":"2012","unstructured":"Kiri\u00a0L Wagstaff. 2012. Machine learning that matters. In Proceedings of the 29th International Coference on International Conference on Machine Learning. 1851\u20131856."},{"key":"e_1_3_2_2_31_1","article-title":"Impact of adding a limitations section to abstracts of systematic reviews on readers\u2019 interpretation: A randomized controlled trial","volume":"14","author":"Yavchitz Am\u00e9lie","year":"2014","unstructured":"Am\u00e9lie Yavchitz, Philippe Ravaud, Sally Hopewell, Gabriel Baron, and Isabelle Boutron. 2014. Impact of adding a limitations section to abstracts of systematic reviews on readers\u2019 interpretation: A randomized controlled trial. BMC Medical Review Methodology 14, 123 (2014).","journal-title":"BMC Medical Review Methodology"}],"event":{"name":"FAccT '22: 2022 ACM Conference on Fairness, Accountability, and Transparency","location":"Seoul Republic of Korea","acronym":"FAccT '22","sponsor":["ACM Association for Computing Machinery"]},"container-title":["2022 ACM Conference on Fairness Accountability and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3533122","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3531146.3533122","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:02:08Z","timestamp":1750186928000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3533122"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,20]]},"references-count":31,"alternative-id":["10.1145\/3531146.3533122","10.1145\/3531146"],"URL":"https:\/\/doi.org\/10.1145\/3531146.3533122","relation":{},"subject":[],"published":{"date-parts":[[2022,6,20]]},"assertion":[{"value":"2022-06-20","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}