{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,24]],"date-time":"2026-09-24T10:31:34Z","timestamp":1790245894315,"version":"4.1.0"},"publisher-location":"New York, NY, USA","reference-count":69,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,4,19]],"date-time":"2023-04-19T00:00:00Z","timestamp":1681862400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,4,19]]},"DOI":"10.1145\/3544548.3581290","type":"proceedings-article","created":{"date-parts":[[2023,4,20]],"date-time":"2023-04-20T04:27:55Z","timestamp":1681964875000},"page":"1-24","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":25,"title":["Model Sketching: Centering Concepts in Early-Stage Machine Learning Model Design"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-3448-5961","authenticated-orcid":false,"given":"Michelle S.","family":"Lam","sequence":"first","affiliation":[{"name":"Computer Science, Stanford University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6175-3039","authenticated-orcid":false,"given":"Zixian","family":"Ma","sequence":"additional","affiliation":[{"name":"Stanford University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5449-7022","authenticated-orcid":false,"given":"Anne","family":"Li","sequence":"additional","affiliation":[{"name":"Stanford University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5895-1365","authenticated-orcid":false,"given":"Izequiel","family":"Freitas","sequence":"additional","affiliation":[{"name":"Stanford University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9371-9441","authenticated-orcid":false,"given":"Dakuo","family":"Wang","sequence":"additional","affiliation":[{"name":"Northeastern University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1520-8894","authenticated-orcid":false,"given":"James A.","family":"Landay","sequence":"additional","affiliation":[{"name":"Computer Science, Stanford University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8020-9434","authenticated-orcid":false,"given":"Michael S.","family":"Bernstein","sequence":"additional","affiliation":[{"name":"Computer Science, Stanford University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,4,19]]},"reference":[{"key":"e_1_3_3_3_1_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v35i4.2513"},{"key":"e_1_3_3_3_2_1","unstructured":"Apple. 2022. Create ML. https:\/\/developer.apple.com\/machine-learning\/create-ml\/"},{"key":"e_1_3_3_3_3_1","unstructured":"Phil Aquilina. 2022. Building Better Moderator Tools. https:\/\/www.reddit.com\/r\/RedditEng\/comments\/uly8s4\/building_better_moderator_tools\/"},{"key":"e_1_3_3_3_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/VAST.2015.7347637"},{"key":"e_1_3_3_3_5_1","volume-title":"Advances in Neural Information Processing Systems, H.\u00a0Larochelle, M.\u00a0Ranzato, R.\u00a0Hadsell, M.F. Balcan, and H.\u00a0Lin (Eds.). Vol.\u00a033. Curran Associates","author":"Brown Tom","year":"1877","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared\u00a0D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, Sandhini Agarwal, Ariel Herbert-Voss, Gretchen Krueger, Tom Henighan, Rewon Child, Aditya Ramesh, Daniel Ziegler, Jeffrey Wu, Clemens Winter, Chris Hesse, Mark Chen, Eric Sigler, Mateusz Litwin, Scott Gray, Benjamin Chess, Jack Clark, Christopher Berner, Sam McCandlish, Alec Radford, Ilya Sutskever, and Dario Amodei. 2020. Language Models are Few-Shot Learners. In Advances in Neural Information Processing Systems, H.\u00a0Larochelle, M.\u00a0Ranzato, R.\u00a0Hadsell, M.F. Balcan, and H.\u00a0Lin (Eds.). Vol.\u00a033. Curran Associates, Inc., 1877\u20131901. https:\/\/proceedings.neurips.cc\/paper\/2020\/file\/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf"},{"key":"e_1_3_3_3_6_1","unstructured":"Bill Buxton. 2010. Sketching User Experiences: Getting the Design Right and the Right Design. Morgan Kaufmann."},{"key":"e_1_3_3_3_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3026044"},{"key":"e_1_3_3_3_8_1","volume-title":"Constructing grounded theory: A practical guide through qualitative analysis","author":"Charmaz Kathy","unstructured":"Kathy Charmaz. 2006. Constructing grounded theory: A practical guide through qualitative analysis. Sage."},{"key":"e_1_3_3_3_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/108844.108850"},{"key":"e_1_3_3_3_10_1","unstructured":"Allen Cypher Daniel\u00a0C. Halbert David Kurlander Henry Lieberman David Maulsby Brad\u00a0A. Myers and Alan Turransky (Eds.). 1993. Watch What I Do: Programming by Demonstration. MIT Press Cambridge MA USA."},{"key":"e_1_3_3_3_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3172944.3172989"},{"key":"e_1_3_3_3_12_1","unstructured":"Siddharth Dangi. 2020. Understanding dwell time to improve LinkedIn feed ranking. https:\/\/engineering.linkedin.com\/blog\/2020\/understanding-feed-dwell-time"},{"key":"e_1_3_3_3_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1879831.1879836"},{"key":"e_1_3_3_3_14_1","doi-asserted-by":"publisher","unstructured":"Dean Eckles. 2022. Algorithmic transparency and assessing effects of algorithmic ranking. https:\/\/doi.org\/10.31235\/osf.io\/c8za6","DOI":"10.31235\/osf.io"},{"key":"e_1_3_3_3_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/604045.604056"},{"key":"e_1_3_3_3_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/642611.642652"},{"key":"e_1_3_3_3_17_1","volume-title":"On-the-fly Machine Learning. In International Conference on New Interfaces for Musical Expression","author":"Fiebrink Rebecca","year":"2009","unstructured":"Rebecca Fiebrink, Dan Trueman, and Perry\u00a0R. Cook. 2009. A Meta-Instrument for Interactive, On-the-fly Machine Learning. In International Conference on New Interfaces for Musical Expression (Pittsburgh, PA) (NIME \u201909)."},{"key":"e_1_3_3_3_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357061"},{"key":"e_1_3_3_3_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3472749.3474734"},{"key":"e_1_3_3_3_20_1","volume-title":"Sketches of Thought","author":"Goel Vinod","unstructured":"Vinod Goel. 1995. Sketches of Thought. MIT Press."},{"key":"e_1_3_3_3_21_1","unstructured":"Google. 2022. Teachable Machine. https:\/\/teachablemachine.withgoogle.com\/"},{"key":"e_1_3_3_3_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3502004"},{"key":"e_1_3_3_3_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/1166253.1166300"},{"key":"e_1_3_3_3_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376177"},{"key":"e_1_3_3_3_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3134688"},{"key":"e_1_3_3_3_26_1","doi-asserted-by":"publisher","unstructured":"Gabriel Ilharco Mitchell Wortsman Ross Wightman Cade Gordon Nicholas Carlini Rohan Taori Achal Dave Vaishaal Shankar Hongseok Namkoong John Miller Hannaneh Hajishirzi Ali Farhadi and Ludwig Schmidt. 2021. OpenCLIP. https:\/\/doi.org\/10.5281\/zenodo.5143773 If you use this software please cite it as below..","DOI":"10.5281\/zenodo.5143773"},{"key":"e_1_3_3_3_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445901"},{"key":"e_1_3_3_3_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3503564"},{"key":"e_1_3_3_3_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3476980"},{"key":"e_1_3_3_3_30_1","doi-asserted-by":"publisher","DOI":"10.1111\/rssa.12576"},{"key":"e_1_3_3_3_31_1","volume-title":"Advances in Neural Information Processing Systems, H.\u00a0Larochelle, M.\u00a0Ranzato, R.\u00a0Hadsell, M.F. Balcan, and H.\u00a0Lin (Eds.). Vol.\u00a033. Curran Associates","author":"Kiela Douwe","year":"2020","unstructured":"Douwe Kiela, Hamed Firooz, Aravind Mohan, Vedanuj Goswami, Amanpreet Singh, Pratik Ringshia, and Davide Testuggine. 2020. The Hateful Memes Challenge: Detecting Hate Speech in Multimodal Memes. In Advances in Neural Information Processing Systems, H.\u00a0Larochelle, M.\u00a0Ranzato, R.\u00a0Hadsell, M.F. Balcan, and H.\u00a0Lin (Eds.). Vol.\u00a033. Curran Associates, Inc., 2611\u20132624. https:\/\/proceedings.neurips.cc\/paper\/2020\/file\/1b84c4cee2b8b3d823b30e2d604b1878-Paper.pdf"},{"key":"e_1_3_3_3_32_1","volume-title":"Proceedings of the 35th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a080)","author":"Kim Been","year":"2018","unstructured":"Been Kim, Martin Wattenberg, Justin Gilmer, Carrie Cai, James Wexler, Fernanda Viegas, and Rory sayres. 2018. Interpretability Beyond Feature Attribution: Quantitative Testing with Concept Activation Vectors (TCAV). In Proceedings of the 35th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a080), Jennifer Dy and Andreas Krause (Eds.). PMLR, 2668\u20132677. https:\/\/proceedings.mlr.press\/v80\/kim18d.html"},{"key":"e_1_3_3_3_33_1","doi-asserted-by":"publisher","DOI":"10.5555\/3524938.3525433"},{"key":"e_1_3_3_3_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2557238"},{"key":"e_1_3_3_3_35_1","volume-title":"Seventeenth Symposium on Usable Privacy and Security (SOUPS 2021","author":"Kumar Deepak","year":"2021","unstructured":"Deepak Kumar, Patrick\u00a0Gage Kelley, Sunny Consolvo, Joshua Mason, Elie Bursztein, Zakir Durumeric, Kurt Thomas, and Michael Bailey. 2021. Designing Toxic Content Classification for a Diversity of Perspectives. In Seventeenth Symposium on Usable Privacy and Security (SOUPS 2021). USENIX Association, 299\u2013318. https:\/\/www.usenix.org\/conference\/soups2021\/presentation\/kumar"},{"key":"e_1_3_3_3_36_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W19-3823"},{"key":"e_1_3_3_3_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3555625"},{"key":"e_1_3_3_3_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/223904.223910"},{"key":"e_1_3_3_3_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/1375761.1375762"},{"key":"e_1_3_3_3_40_1","doi-asserted-by":"publisher","DOI":"10.5555\/2002472.2002491"},{"key":"e_1_3_3_3_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3361118"},{"key":"e_1_3_3_3_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359174"},{"key":"e_1_3_3_3_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300356"},{"key":"e_1_3_3_3_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3357236.3395454"},{"key":"e_1_3_3_3_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287567"},{"key":"e_1_3_3_3_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/2998181.2998331"},{"key":"e_1_3_3_3_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274405"},{"key":"e_1_3_3_3_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/1866029.1866038"},{"key":"e_1_3_3_3_49_1","doi-asserted-by":"publisher","DOI":"10.5555\/1953048.2078195"},{"key":"e_1_3_3_3_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3449205"},{"key":"e_1_3_3_3_51_1","doi-asserted-by":"publisher","unstructured":"Alec Radford Jong\u00a0Wook Kim Chris Hallacy Aditya Ramesh Gabriel Goh Sandhini Agarwal Girish Sastry Amanda Askell Pamela Mishkin Jack Clark Gretchen Krueger and Ilya Sutskever. 2021. Learning Transferable Visual Models From Natural Language Supervision. https:\/\/doi.org\/10.48550\/ARXIV.2103.00020","DOI":"10.48550\/ARXIV.2103.00020"},{"key":"e_1_3_3_3_52_1","volume-title":"Interactive machine teaching: a human-centered approach to building machine-learned models. Human\u2013Computer Interaction 35, 5-6","author":"Ramos Gonzalo","year":"2020","unstructured":"Gonzalo Ramos, Christopher Meek, Patrice Simard, Jina Suh, and Soroush Ghorashi. 2020. Interactive machine teaching: a human-centered approach to building machine-learned models. Human\u2013Computer Interaction 35, 5-6 (2020), 413\u2013451."},{"key":"e_1_3_3_3_53_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01042"},{"key":"e_1_3_3_3_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173606"},{"key":"e_1_3_3_3_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445518"},{"key":"e_1_3_3_3_56_1","doi-asserted-by":"publisher","unstructured":"Emily Sheng Kai-Wei Chang Premkumar Natarajan and Nanyun Peng. 2019. The Woman Worked as a Babysitter: On Biases in Language Generation. In EMNLP\/IJCNLP (1). 3405\u20133410. https:\/\/doi.org\/10.18653\/v1\/D19-1339","DOI":"10.18653\/v1"},{"key":"e_1_3_3_3_57_1","unstructured":"Patrice Simard Saleema Amershi Max Chickering Alicia Edelman\u00a0Pelton Soroush Ghorashi Chris Meek Gonzalo Ramos Jina Suh Johan Verwey Mo Wang and John Wernsing. 2017. Machine Teaching: A New Paradigm for Building Machine Learning Systems. Technical Report MSR-TR-2017-26. https:\/\/www.microsoft.com\/en-us\/research\/publication\/machine-teaching-new-paradigm-building-machine-learning-systems\/"},{"key":"e_1_3_3_3_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/257089.257255"},{"key":"e_1_3_3_3_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/1124772.1124960"},{"key":"e_1_3_3_3_60_1","unstructured":"Leandro von Werra. 2021. distilbert-imdb. https:\/\/huggingface.co\/lvwerra\/distilbert-imdb"},{"key":"e_1_3_3_3_61_1","doi-asserted-by":"publisher","DOI":"10.1177\/154193120204600513"},{"key":"e_1_3_3_3_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359313"},{"key":"e_1_3_3_3_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517582"},{"key":"e_1_3_3_3_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3319616"},{"key":"e_1_3_3_3_65_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.destud.2005.04.005"},{"key":"e_1_3_3_3_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300415"},{"key":"e_1_3_3_3_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376301"},{"key":"e_1_3_3_3_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3392826"},{"key":"e_1_3_3_3_69_1","doi-asserted-by":"publisher","unstructured":"Ron Zhu. 2020. Enhance Multimodal Transformer With External Label And In-Domain Pretrain: Hateful Meme Challenge Winning Solution. https:\/\/doi.org\/10.48550\/ARXIV.2012.08290","DOI":"10.48550\/ARXIV.2012.08290"}],"event":{"name":"CHI '23: CHI Conference on Human Factors in Computing Systems","location":"Hamburg Germany","acronym":"CHI '23","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3544548.3581290","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3544548.3581290","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:37:30Z","timestamp":1750178250000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3544548.3581290"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,4,19]]},"references-count":69,"alternative-id":["10.1145\/3544548.3581290","10.1145\/3544548"],"URL":"https:\/\/doi.org\/10.1145\/3544548.3581290","relation":{},"subject":[],"published":{"date-parts":[[2023,4,19]]},"assertion":[{"value":"2023-04-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}