{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,21]],"date-time":"2026-05-21T09:55:41Z","timestamp":1779357341588,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":69,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,5,11]],"date-time":"2024-05-11T00:00:00Z","timestamp":1715385600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-sa\/4.0\/"}],"funder":[{"name":"CMU K&L Gates Presidential Scholarship"},{"name":"CMU Digital Transformation and Innovation Center"},{"name":"Center for Advancing Safety of Machine Intelligence"},{"name":"CMU Block Center for Technology and Society Award","award":["53680.1.5007718"],"award-info":[{"award-number":["53680.1.5007718"]}]},{"DOI":"10.13039\/https:\/\/doi.org\/10.13039\/100000001","name":"NSF (National Science Foundation) Graduate Research Fellowship","doi-asserted-by":"publisher","id":[{"id":"10.13039\/https:\/\/doi.org\/10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,5,11]]},"DOI":"10.1145\/3613904.3642849","type":"proceedings-article","created":{"date-parts":[[2024,5,11]],"date-time":"2024-05-11T08:38:25Z","timestamp":1715416705000},"page":"1-22","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":42,"title":["The Situate AI Guidebook: Co-Designing a Toolkit to Support Multi-Stakeholder, Early-stage Deliberations Around Public Sector AI Proposals"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0000-6937-6370","authenticated-orcid":false,"given":"Anna","family":"Kawakami","sequence":"first","affiliation":[{"name":"Human-Computer Interaction Institute, Carnegie Mellon University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9282-9921","authenticated-orcid":false,"given":"Amanda","family":"Coston","sequence":"additional","affiliation":[{"name":"Microsoft Research, United States and Heinz College and Machine Learning Department, Carnegie Mellon University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7271-9100","authenticated-orcid":false,"given":"Haiyi","family":"Zhu","sequence":"additional","affiliation":[{"name":"Human Computer Interaction Institute, Carnegie Mellon University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3710-4076","authenticated-orcid":false,"given":"Hoda","family":"Heidari","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6730-922X","authenticated-orcid":false,"given":"Kenneth","family":"Holstein","sequence":"additional","affiliation":[{"name":"Human-Computer Interaction Institute, Carnegie Mellon University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,5,11]]},"reference":[{"key":"e_1_3_3_3_1_1","unstructured":"2022. Community jury - Azure Application Architecture Guide. https:\/\/learn.microsoft.com\/en-us\/azure\/architecture\/guide\/responsible-innovation\/community-jury\/"},{"key":"e_1_3_3_3_2_1","unstructured":"2023. Algorithmic Impact Assessment tool | Government of Canada. https:\/\/www.canada.ca\/en\/government\/system\/digital-government\/digital-government-innovations\/responsible-use-ai\/algorithmic-impact-assessment.html"},{"key":"e_1_3_3_3_3_1","unstructured":"2023. A guiding framework to vetting public sector technology vendors. https:\/\/www.fordfoundation.org\/work\/learning\/research-reports\/a-guiding-framework-to-vetting-public-sector-technology-vendors\/"},{"key":"e_1_3_3_3_4_1","doi-asserted-by":"crossref","unstructured":"J\u00a0Khadijah Abdurahman. 2021. Calculating the Souls of Black Folk: Predictive Analytics in the New York City Administration for Children\u2019s Services. In Colum. J. Race & L. Forum Vol.\u00a011. HeinOnline 75.","DOI":"10.52214\/cjrl.v11i4.8741"},{"key":"e_1_3_3_3_5_1","unstructured":"Accenture. 2018. Public services in the era of artificial intelligence. https:\/\/www.accenture.com\/us-en\/services\/public-service\/artificial-intelligence"},{"key":"e_1_3_3_3_6_1","volume-title":"Artificial intelligence and fraud detection. Innovative Technology at the Interface of Finance and Operations","author":"Bao Yang","year":"2022","unstructured":"Yang Bao, Gilles Hilary, and Bin Ke. 2022. Artificial intelligence and fraud detection. Innovative Technology at the Interface of Finance and Operations: Volume I (2022), 223\u2013247."},{"key":"e_1_3_3_3_7_1","doi-asserted-by":"publisher","DOI":"10.1136\/bmjopen-2012-001667"},{"key":"e_1_3_3_3_8_1","volume-title":"Reflecting on reflexive thematic analysis. Qualitative research in sport, exercise and health 11, 4","author":"Braun Virginia","year":"2019","unstructured":"Virginia Braun and Victoria Clarke. 2019. Reflecting on reflexive thematic analysis. Qualitative research in sport, exercise and health 11, 4 (2019), 589\u2013597."},{"key":"e_1_3_3_3_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300271"},{"key":"e_1_3_3_3_10_1","volume-title":"How Child Welfare Workers Reduce Racial Disparities in Algorithmic Decisions. In CHI Conference on Human Factors in Computing Systems. 1\u201322","author":"Cheng Hao-Fei","year":"2022","unstructured":"Hao-Fei Cheng, Logan Stapleton, Anna Kawakami, Venkatesh Sivaraman, Yanghuidi Cheng, Diana Qing, Adam Perer, Kenneth Holstein, Zhiwei\u00a0Steven Wu, and Haiyi Zhu. 2022. How Child Welfare Workers Reduce Racial Disparities in Algorithmic Decisions. In CHI Conference on Human Factors in Computing Systems. 1\u201322."},{"key":"e_1_3_3_3_11_1","volume-title":"Conference on Fairness, Accountability and Transparency. PMLR, 134\u2013148","author":"Chouldechova Alexandra","year":"2018","unstructured":"Alexandra Chouldechova, Diana Benavides-Prado, Oleksandr Fialko, and Rhema Vaithianathan. 2018. A case study of algorithm-assisted decision making in child maltreatment hotline screening decisions. In Conference on Fairness, Accountability and Transparency. PMLR, 134\u2013148."},{"key":"e_1_3_3_3_12_1","first-page":"1","article-title":"A case study of algorithm-assisted decision making in child maltreatment hotline screening decisions","volume":"81","author":"Chouldechova Alexandra","year":"2018","unstructured":"Alexandra Chouldechova, Emily Putnam-Hornstein, Suzanne Dworak-Peck, Diana Benavides-Prado, Oleksandr Fialko, Rhema Vaithianathan, Sorelle\u00a0A Friedler, and Christo Wilson. 2018. A case study of algorithm-assisted decision making in child maltreatment hotline screening decisions. Proceedings of Machine Learning Research 81 (2018), 1\u201315. http:\/\/proceedings.mlr.press\/v81\/chouldechova18a.html","journal-title":"Proceedings of Machine Learning Research"},{"key":"e_1_3_3_3_13_1","volume-title":"A Validity Perspective on Evaluating the Justified Use of Data-driven Decision-making Algorithms. arXiv preprint arXiv:2206.14983","author":"Coston Amanda","year":"2022","unstructured":"Amanda Coston, Anna Kawakami, Haiyi Zhu, Ken Holstein, and Hoda Heidari. 2022. A Validity Perspective on Evaluating the Justified Use of Data-driven Decision-making Algorithms. arXiv preprint arXiv:2206.14983 (2022)."},{"key":"e_1_3_3_3_14_1","volume-title":"Ethical analysis: Predictive risk models at call screening for Allegheny County. Alleghany County Analytics","author":"Dare Tim","year":"2017","unstructured":"Tim Dare and Eileen Gambrill. 2017. Ethical analysis: Predictive risk models at call screening for Allegheny County. Alleghany County Analytics (2017)."},{"key":"e_1_3_3_3_15_1","volume-title":"Carmen\u00a0L Lewis","author":"Elwyn Glyn","year":"2013","unstructured":"Glyn Elwyn, Isabelle Scholl, Caroline Tietbohl, Mala Mann, Adrian\u00a0GK Edwards, Catharine Clay, France L\u00e9gar\u00e9, Trudy van\u00a0der Weijden, Carmen\u00a0L Lewis, Richard\u00a0M Wexler, 2013. \u201cMany miles to go\u2026\u201d: a systematic review of the implementation of patient decision support interventions into routine clinical practice. BMC medical informatics and decision making 13 (2013), 1\u201310."},{"key":"e_1_3_3_3_16_1","volume-title":"Public Law Research Paper20-54","author":"Engstrom David\u00a0Freeman","year":"2020","unstructured":"David\u00a0Freeman Engstrom, Daniel\u00a0E Ho, Catherine\u00a0M Sharkey, and Mariano-Florentino Cu\u00e9llar. 2020. Government by algorithm: Artificial intelligence in federal administrative agencies. NYU School of Law, Public Law Research Paper20-54 (2020)."},{"key":"e_1_3_3_3_17_1","volume-title":"Automating inequality: How high-tech tools profile, police, and punish the poor","author":"Eubanks Virginia","unstructured":"Virginia Eubanks. 2018. Automating inequality: How high-tech tools profile, police, and punish the poor. St. Martin\u2019s Press."},{"key":"e_1_3_3_3_18_1","volume-title":"Data Ethics Decision Aid (DEDA): a dialogical framework for ethical inquiry of AI and data projects in the Netherlands. Ethics and Information Technology","author":"Franzke Aline\u00a0Shakti","year":"2021","unstructured":"Aline\u00a0Shakti Franzke, Iris Muis, and Mirko\u00a0Tobias Sch\u00e4fer. 2021. Data Ethics Decision Aid (DEDA): a dialogical framework for ethical inquiry of AI and data projects in the Netherlands. Ethics and Information Technology (2021), 1\u201317."},{"key":"e_1_3_3_3_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3458723"},{"key":"e_1_3_3_3_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3593013.3594081"},{"key":"e_1_3_3_3_21_1","volume-title":"Barriers and facilitators to implementing shared decision-making in clinical practice: a systematic review of health professionals","author":"Gravel Karine","year":"2006","unstructured":"Karine Gravel, France L\u00e9gar\u00e9, and Ian\u00a0D Graham. 2006. Barriers and facilitators to implementing shared decision-making in clinical practice: a systematic review of health professionals\u2019 perceptions. Implementation science 1 (2006), 1\u201312."},{"key":"e_1_3_3_3_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372869"},{"key":"e_1_3_3_3_23_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.clsr.2022.105681"},{"key":"e_1_3_3_3_24_1","volume-title":"An algorithm that screens for child neglect raises concerns. AP News","author":"Ho Sally","year":"2022","unstructured":"Sally Ho and Garance Burke. 2022. An algorithm that screens for child neglect raises concerns. AP News (2022)."},{"key":"e_1_3_3_3_25_1","volume-title":"Child welfare algorithm faces Justice Department scrutiny. AP News","author":"Ho Sally","year":"2023","unstructured":"Sally Ho and Garance Burke. 2023. Child welfare algorithm faces Justice Department scrutiny. AP News (2023)."},{"key":"e_1_3_3_3_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3027385.3027451"},{"key":"e_1_3_3_3_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3419249.3420149"},{"key":"e_1_3_3_3_28_1","volume-title":"Oregon is dropping an artificial intelligence tool used in child welfare system. Link: https:\/\/www. npr. org\/2022\/06\/02\/1102661376\/oregon-drops-artificial-intelligence-child-abuse-cases","author":"June NPR","year":"2022","unstructured":"NPR June. 2022. Oregon is dropping an artificial intelligence tool used in child welfare system. Link: https:\/\/www. npr. org\/2022\/06\/02\/1102661376\/oregon-drops-artificial-intelligence-child-abuse-cases (2022)."},{"key":"e_1_3_3_3_29_1","doi-asserted-by":"crossref","unstructured":"Anna Kawakami Amanda Coston Hoda Heidari Kenneth Holstein and Haiyi Zhu. 2023. Studying Up Public Sector AI: How Networks of Power Relations Shape Agency Decisions Around AI Design and Use. (2023).","DOI":"10.1145\/3686989"},{"key":"e_1_3_3_3_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517439"},{"key":"e_1_3_3_3_31_1","volume-title":"Worker-AI Interface Design Concepts. In Designing Interactive Systems Conference. 454\u2013470","author":"Kawakami Anna","year":"2022","unstructured":"Anna Kawakami, Venkatesh Sivaraman, Logan Stapleton, Hao-Fei Cheng, Adam Perer, Zhiwei\u00a0Steven Wu, Haiyi Zhu, and Kenneth Holstein. 2022. \u201cWhy Do I Care What\u2019s Similar?\u201d Probing Challenges in AI-Assisted Child Welfare Decision-Making through Worker-AI Interface Design Concepts. In Designing Interactive Systems Conference. 454\u2013470."},{"key":"e_1_3_3_3_32_1","volume-title":"Human decisions and machine predictions. The quarterly journal of economics 133, 1","author":"Kleinberg Jon","year":"2018","unstructured":"Jon Kleinberg, Himabindu Lakkaraju, Jure Leskovec, Jens Ludwig, and Sendhil Mullainathan. 2018. Human decisions and machine predictions. The quarterly journal of economics 133, 1 (2018), 237\u2013293."},{"key":"e_1_3_3_3_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445938"},{"key":"e_1_3_3_3_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145249"},{"key":"e_1_3_3_3_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580882"},{"key":"e_1_3_3_3_36_1","doi-asserted-by":"publisher","DOI":"10.1146\/annurev-lawsocsci-041221-023808"},{"key":"e_1_3_3_3_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376445"},{"key":"e_1_3_3_3_38_1","doi-asserted-by":"publisher","unstructured":"Shefeh\u00a0Prisilia Mbuy and Marco Ortolani. 2022. Algorithmic Impact Assessment for an Ethical Use of AI in SMEs. https:\/\/doi.org\/10.14236\/ewic\/HCI2022.34","DOI":"10.14236\/ewic"},{"key":"e_1_3_3_3_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287596"},{"key":"e_1_3_3_3_40_1","volume-title":"Assembling accountability: algorithmic impact assessment for the public interest. Available at SSRN 3877437","author":"Moss Emanuel","year":"2021","unstructured":"Emanuel Moss, Elizabeth\u00a0Anne Watkins, Ranjit Singh, Madeleine\u00a0Clare Elish, and Jacob Metcalf. 2021. Assembling accountability: algorithmic impact assessment for the public interest. Available at SSRN 3877437 (2021)."},{"key":"e_1_3_3_3_41_1","volume-title":"Public sector vs. private sector: What\u2019s the difference. The Balance Small Business","author":"Murray J","year":"2021","unstructured":"J Murray. 2021. Public sector vs. private sector: What\u2019s the difference. The Balance Small Business (2021)."},{"key":"e_1_3_3_3_42_1","volume-title":"How to recognize AI snake oil. Arthur Miller Lecture on Science and Ethics","author":"Narayanan Arvind","year":"2019","unstructured":"Arvind Narayanan. 2019. How to recognize AI snake oil. Arthur Miller Lecture on Science and Ethics (2019)."},{"key":"e_1_3_3_3_43_1","unstructured":"NIST. 2023. NIST AI RMF Playbook. https:\/\/airc.nist.gov\/AI_RMF_Knowledge_Base\/Playbook"},{"key":"e_1_3_3_3_44_1","unstructured":"NIST. 2023. NIST Risk Management Framework. https:\/\/csrc.nist.gov\/projects\/risk-management\/about-rmf"},{"key":"e_1_3_3_3_45_1","volume-title":"Predicting the future\u2014big data, machine learning, and clinical medicine. The New England journal of medicine 375, 13","author":"Obermeyer Ziad","year":"2016","unstructured":"Ziad Obermeyer and Ezekiel\u00a0J Emanuel. 2016. Predicting the future\u2014big data, machine learning, and clinical medicine. The New England journal of medicine 375, 13 (2016), 1216."},{"key":"e_1_3_3_3_46_1","doi-asserted-by":"publisher","DOI":"10.1071\/AH09845"},{"key":"e_1_3_3_3_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287567"},{"key":"e_1_3_3_3_48_1","unstructured":"Allied\u00a0Media Projects. 2018. A People\u2019s Guide To Tech. https:\/\/alliedmedia.org\/resources\/peoples-guide-to-ai"},{"key":"e_1_3_3_3_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533231"},{"key":"e_1_3_3_3_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533158"},{"key":"e_1_3_3_3_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3449081"},{"key":"e_1_3_3_3_52_1","volume-title":"Algorithmic Impact Assessments: A Practical Framework for Public Agency. AI Now","author":"Reisman Dillon","year":"2018","unstructured":"Dillon Reisman, Jason Schultz, Kate Crawford, and Meredith Whittaker. 2018. Algorithmic Impact Assessments: A Practical Framework for Public Agency. AI Now (2018)."},{"key":"e_1_3_3_3_53_1","volume-title":"A methodology for creating AI FactSheets. arXiv preprint arXiv:2006.13796","author":"Richards John","year":"2020","unstructured":"John Richards, David Piorkowski, Michael Hind, Stephanie Houde, and Aleksandra Mojsilovi\u0107. 2020. A methodology for creating AI FactSheets. arXiv preprint arXiv:2006.13796 (2020)."},{"key":"e_1_3_3_3_54_1","volume-title":"Torn apart: how the child welfare system destroys black families\u2013and how abolition can build a safer world","author":"Roberts Dorothy","unstructured":"Dorothy Roberts. 2022. Torn apart: how the child welfare system destroys black families\u2013and how abolition can build a safer world. Basic Books."},{"key":"e_1_3_3_3_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445748"},{"key":"e_1_3_3_3_56_1","volume-title":"Family surveillance by algorithm: The rapidly spreading tools few have heard of","author":"Samant Anjana","year":"2021","unstructured":"Anjana Samant, Aaron Horowitz, Kath Xu, and Sophie Beiers. 2021. Family surveillance by algorithm: The rapidly spreading tools few have heard of. American Civil Liberties Union (ACLU) (2021). https:\/\/www.aclu.org\/sites\/default\/files\/field_document\/2021.09.28a_family_surveillance_by_algorithm.pdf"},{"key":"e_1_3_3_3_57_1","volume-title":"Family surveillance by algorithm: The rapidly spreading tools few have heard of","author":"Samant Anjana","year":"2021","unstructured":"Anjana Samant, Aaron Horowitz, Kath Xu, and Sophie Beiers. 2021. Family surveillance by algorithm: The rapidly spreading tools few have heard of. American Civil Liberties Union (ACLU)(2021)."},{"key":"e_1_3_3_3_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376229"},{"key":"e_1_3_3_3_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581308"},{"key":"e_1_3_3_3_60_1","doi-asserted-by":"publisher","DOI":"10.1177\/0162243912436985"},{"key":"e_1_3_3_3_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533177"},{"key":"e_1_3_3_3_62_1","doi-asserted-by":"crossref","unstructured":"Marie Utterberg\u00a0Mod\u00e9n Martin Tallvid Johan Lundin and Berner Lindstr\u00f6m. 2021. Intelligent tutoring systems: Why teachers abandoned a technology aimed at automating teaching processes. (2021).","DOI":"10.24251\/HICSS.2021.186"},{"key":"e_1_3_3_3_63_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.amepre.2013.04.022"},{"key":"e_1_3_3_3_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174014"},{"key":"e_1_3_3_3_65_1","volume-title":"Against Predictive Optimization: On the Legitimacy of Decision-Making Algorithms that Optimize Predictive Accuracy. Available at SSRN","author":"Wang Angelina","year":"2022","unstructured":"Angelina Wang, Sayash Kapoor, Solon Barocas, and Arvind Narayanan. 2022. Against Predictive Optimization: On the Legitimacy of Decision-Making Algorithms that Optimize Predictive Accuracy. Available at SSRN (2022)."},{"key":"e_1_3_3_3_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/3579621"},{"key":"e_1_3_3_3_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300468"},{"key":"e_1_3_3_3_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3563657.3596058"},{"key":"e_1_3_3_3_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580900"}],"event":{"name":"CHI '24: CHI Conference on Human Factors in Computing Systems","location":"Honolulu HI USA","acronym":"CHI '24","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction","SIGACCESS ACM Special Interest Group on Accessible Computing"]},"container-title":["Proceedings of the CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3613904.3642849","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3613904.3642849","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:44:30Z","timestamp":1750290270000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3613904.3642849"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5,11]]},"references-count":69,"alternative-id":["10.1145\/3613904.3642849","10.1145\/3613904"],"URL":"https:\/\/doi.org\/10.1145\/3613904.3642849","relation":{},"subject":[],"published":{"date-parts":[[2024,5,11]]},"assertion":[{"value":"2024-05-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}