{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,13]],"date-time":"2026-06-13T07:07:41Z","timestamp":1781334461823,"version":"3.54.1"},"publisher-location":"New York, NY, USA","reference-count":116,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,5,21]],"date-time":"2022-05-21T00:00:00Z","timestamp":1653091200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"Natural Sciences and Engineering Research Council of Canada (NSERC)","award":["RGPIN2021-03538"],"award-info":[{"award-number":["RGPIN2021-03538"]}]},{"name":"NSF","award":["1813598, 2131477"],"award-info":[{"award-number":["1813598, 2131477"]}]},{"name":"Carnegie Mellon University (CMU)"},{"name":"Department of Defense","award":["FA8702-15-D-0002"],"award-info":[{"award-number":["FA8702-15-D-0002"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,5,21]]},"DOI":"10.1145\/3510003.3510209","type":"proceedings-article","created":{"date-parts":[[2022,7,5]],"date-time":"2022-07-05T22:42:59Z","timestamp":1657060979000},"page":"413-425","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":122,"title":["Collaboration challenges in building ML-enabled systems"],"prefix":"10.1145","author":[{"given":"Nadia","family":"Nahar","sequence":"first","affiliation":[{"name":"Carnegie Mellon University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shurui","family":"Zhou","sequence":"additional","affiliation":[{"name":"University of Toronto, Toronto, Ontario, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Grace","family":"Lewis","sequence":"additional","affiliation":[{"name":"Carnegie Mellon Software Engineering Institute"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Christian","family":"K\u00e4stner","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2022,7,5]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proc. Int'l Conf. Product-Focused Software Process Improvement, 153--167","author":"Aho T.","unstructured":"Aho, T., Sievi-Korte, O., Kilamo, T., Yaman, S. and Mikkonen, T., 2020. Demystifying data science projects: A look on the people and process of data science today. In Proc. Int'l Conf. Product-Focused Software Process Improvement, 153--167."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"crossref","unstructured":"Akkerman S.F. and Bakker A. 2011. Boundary Crossing and Boundary Objects. Review of educational research. 81 2 132--169.","DOI":"10.3102\/0034654311404435"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"crossref","unstructured":"Akkiraju R. Sinha V. Xu A. Mahmud J. Gundecha P. Liu Z. Liu X. and Schumacher J. 2020. Characterizing Machine Learning Processes: A Maturity Framework. Business Process Management 17--31.","DOI":"10.1007\/978-3-030-58666-9_2"},{"key":"e_1_3_2_1_4_1","volume-title":"Building Machine Learning Powered Applications: Going from Idea to Product","author":"Ameisen E.","unstructured":"Ameisen, E. 2020. Building Machine Learning Powered Applications: Going from Idea to Product. O'Reilly Media, Inc."},{"key":"e_1_3_2_1_5_1","volume-title":"Proc. of 41st Int'l Conf. on Software Engineering: Software Engineering in Practice (ICSE-SEIP), 291--300","author":"Amershi S.","unstructured":"Amershi, S. et al. 2019. Software Engineering for Machine Learning: A Case Study. In Proc. of 41st Int'l Conf. on Software Engineering: Software Engineering in Practice (ICSE-SEIP), 291--300."},{"key":"e_1_3_2_1_6_1","volume-title":"Proc. of 33rd Conf. on Human Factors in Computing Systems, 337--346","author":"Amershi S.","unstructured":"Amershi, S., Chickering, M., Drucker, S.M., Lee, B., Simard, P. and Suh, J. 2015. ModelTracker: Redesigning Performance Analysis Tools for Machine Learning. In Proc. of 33rd Conf. on Human Factors in Computing Systems, 337--346."},{"key":"e_1_3_2_1_7_1","volume-title":"Proc. of CHI Conf. on Human Factors in Computing Systems, 1--13","author":"Amershi S.","unstructured":"Amershi, S. et al. 2019. Guidelines for Human-AI Interaction. In Proc. of CHI Conf. on Human Factors in Computing Systems, 1--13."},{"key":"e_1_3_2_1_8_1","volume-title":"et al","author":"Arnold M.","year":"2019","unstructured":"Arnold, M. et al. 2019. FactSheets: Increasing trust in AI services through supplier's declarations of conformity. IBM Journal of Research and Development, 63."},{"key":"e_1_3_2_1_9_1","volume-title":"Proc. Euromicro Conf. Software Engineering and Advanced Applications (SEAA), 50--59","author":"Arpteg A.","unstructured":"Arpteg, A., Brinne, B., Crnkovic-Friis, L. and Bosch, J. 2018. Software Engineering Challenges of Deep Learning. In Proc. Euromicro Conf. Software Engineering and Advanced Applications (SEAA), 50--59."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3453444"},{"key":"e_1_3_2_1_11_1","unstructured":"Bass L. Clements P. and Kazman R. 1998. Software Architecture in Practice. Addison-Wesley Longman Publishing Co. Inc."},{"key":"e_1_3_2_1_12_1","volume-title":"Proc. Int'l Conf. Global Software Engineering, 31--40","author":"Bass M.","unstructured":"Bass, M., Herbsleb, J.D. and Lescher, C. 2009. A Coordination Risk Analysis Method for Multi-site Projects: Experience Report. In Proc. Int'l Conf. Global Software Engineering, 31--40."},{"key":"e_1_3_2_1_13_1","volume-title":"Proc. Int'l Conf. Knowledge Discovery and Data Mining, 1387--1395","author":"Baylor D.","unstructured":"Baylor, D., Breck, E., Cheng, H.T., Fiedel, N., Foo, C.Y. et al. 2017. TFX: A TensorFlow-Based Production-Scale Machine Learning Platform. In Proc. Int'l Conf. Knowledge Discovery and Data Mining, 1387--1395."},{"key":"e_1_3_2_1_14_1","volume-title":"Proc. Int'l Conf. Knowledge Discovery & Data Mining, 1743--1751","author":"Bernardi L.","unstructured":"Bernardi, L., Mavridis, T. and Estevez, P. 2019. 150 successful machine learning models. In Proc. Int'l Conf. Knowledge Discovery & Data Mining, 1743--1751."},{"key":"e_1_3_2_1_15_1","volume-title":"Proc. of Conf. on Fairness, Accountability, and Transparency, 648--657","author":"Bhatt U.","unstructured":"Bhatt, U., Xiang, A., Sharma, S., Weller, A., Taly, A., Jia, Y., Ghosh, J., Puri, R., Moura, J.M.F. and Eckersley, P. 2020. Explainable machine learning in deployment. In Proc. of Conf. on Fairness, Accountability, and Transparency, 648--657."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3447245"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.2991\/jase.d.190131.001"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"crossref","unstructured":"Bosch J. Olsson H.H. and Crnkovic I. 2021. Engineering AI Systems: A Research Agenda. Artificial Intelligence Paradigms for Smart Cyber-Physical Systems. IGI Global. 1--19.","DOI":"10.4018\/978-1-7998-5101-1.ch001"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1023980212097"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2020.110542"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"crossref","unstructured":"Brandst\u00e4dter S. and Sonntag K. 2016. Interdisciplinary Collaboration. Advances in Ergonomic Design of Systems Products and Processes 395--409.","DOI":"10.1007\/978-3-662-48661-0_26"},{"key":"e_1_3_2_1_22_1","volume-title":"Software Engineering: Modern Approaches","author":"Braude J","year":"2011","unstructured":"Braude, Eric J and Bernstein, Michael E. 2011. Software Engineering: Modern Approaches 2nd Edition. Wiley. ISBN-13: 978-0471692089.","edition":"2"},{"key":"e_1_3_2_1_23_1","volume-title":"Proc. of Int'l Conf. on Big Data (Big Data), 1123--1132","author":"Breck E.","unstructured":"Breck, E., Cai, S., Nielsen, E., Salib, M. and Sculley, D. 2017. The ML test score: A rubric for ML production readiness and technical debt reduction. In Proc. of Int'l Conf. on Big Data (Big Data), 1123--1132."},{"key":"e_1_3_2_1_24_1","volume-title":"Factors that facilitate or inhibit interdisciplinary collaboration within a professional bureaucracy","author":"Brown G.F.C.","unstructured":"Brown, G.F.C. 1995. Factors that facilitate or inhibit interdisciplinary collaboration within a professional bureaucracy. University of Arkansas."},{"key":"e_1_3_2_1_25_1","volume-title":"Proc. Human-Computer Interaction. 3, CSCW, 1--24","author":"Cai C.J.","unstructured":"Cai, C.J., Winter, S., Steiner, D., Wilcox, L. and Terry, M. 2019. \"hello AI\": Uncovering the onboarding needs of medical practitioners for human-AI collaborative decision-making. In Proc. Human-Computer Interaction. 3, CSCW, 1--24."},{"key":"e_1_3_2_1_26_1","volume-title":"Proc. Conf. Computer Supported Cooperative Work (CSCW), 353--362","author":"Cataldo M.","unstructured":"Cataldo, M. et al. 2006. Identification of Coordination Requirements: Implications for the Design of Collaboration and Awareness Tools. In Proc. Conf. Computer Supported Cooperative Work (CSCW), 353--362."},{"key":"e_1_3_2_1_27_1","volume-title":"Proc. Conf. Human Factors in Computing Systems (CHI), 1--12","author":"Chattopadhyay S.","unstructured":"Chattopadhyay, S., Prasad, I., Henley, A.Z., Sarma, A. and Barik, T. 2020. What's Wrong with Computational Notebooks? Pain Points, Needs, and Design Opportunities. In Proc. Conf. Human Factors in Computing Systems (CHI), 1--12."},{"key":"e_1_3_2_1_28_1","volume-title":"Proc. Int'l Conf. Software Quality, Reliability and Security (QRS), 313--324","author":"Cheng D.","unstructured":"Cheng, D., Cao, C., Xu, C. and Ma, X. 2018. Manifesting Bugs in Machine Learning Code: An Explorative Study with Mutation Testing. In Proc. Int'l Conf. Software Quality, Reliability and Security (QRS), 313--324."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"crossref","unstructured":"Chen Z. Cao Y. Liu Y. Wang H. Xie T. and Liu X. 2020. Understanding Challenges in Deploying Deep Learning Based Software: An Empirical Study. arXiv 2005.00760.","DOI":"10.1145\/3368089.3409759"},{"key":"e_1_3_2_1_30_1","first-page":"28","article-title":"How Do Committees Invent","volume":"14","author":"Conway M.E.","year":"1968","unstructured":"Conway, M.E. 1968. How Do Committees Invent? Datamation. 14, 4, 28--31.","journal-title":"Datamation."},{"key":"e_1_3_2_1_31_1","volume-title":"Proc. Int'l Symposium Foundations of Software Engineering (FSE), 1--11","author":"Cossette B.E.","unstructured":"Cossette, B.E. and Walker, R.J. 2012. Seeking the Ground Truth: A Retroactive Study on the Evolution and Migration of Software Libraries. In Proc. Int'l Symposium Foundations of Software Engineering (FSE), 1--11."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/50087.50089"},{"key":"e_1_3_2_1_33_1","volume-title":"Proc. Conf. Computer Supported Cooperative Work (CSCW), 1277--1286","author":"Dabbish L.","unstructured":"Dabbish, L., Stuart, C., Tsay, J. and Herbsleb, J. 2012. Social Coding in GitHub: Transparency and Collaboration in an Open Software Repository. In Proc. Conf. Computer Supported Cooperative Work (CSCW), 1277--1286."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-021-09993-1"},{"key":"e_1_3_2_1_35_1","volume-title":"An Exploratory Study in Fintech. arXiv","author":"Haakman M.","year":"2010","unstructured":"Haakman, M., Cruz, L., Huijgens, H. and van Deursen, A. 2020. AI Lifecycle Models Need To Be Revised. An Exploratory Study in Fintech. arXiv 2010.02716."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.3316\/QRJ1102063"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300500"},{"key":"e_1_3_2_1_38_1","volume-title":"Proc. Int'l Conf. Software Engineering (ICSE), 85--95","author":"Herbsleb J.D.","unstructured":"Herbsleb, J.D. and Grinter, R.E. 1999. Splitting the Organization and Integrating the Code: Conway's Law Revisited. In Proc. Int'l Conf. Software Engineering (ICSE), 85--95."},{"key":"e_1_3_2_1_39_1","volume-title":"Proc. Conf. Human Factors in Computing (CHI) Systems, 1--16","author":"Holstein K.","unstructured":"Holstein, K. et al.. 2019. Improving Fairness in Machine Learning Systems: What Do Industry Practitioners Need? In Proc. Conf. Human Factors in Computing (CHI) Systems, 1--16."},{"key":"e_1_3_2_1_40_1","volume-title":"Proc. Conf. on AI, Ethics, and Society, 134--145","author":"Hopkins A.","unstructured":"Hopkins, A. and Booth, S. 2021. Machine learning practices outside big tech: How resource constraints challenge responsible development. In Proc. Conf. on AI, Ethics, and Society, 134--145."},{"key":"e_1_3_2_1_41_1","volume-title":"European Conf. on Information Systems, 92","author":"Hukkelberg I.","unstructured":"Hukkelberg, I. and Rolland, K. 2020. Exploring Machine Learning in a Large Governmental Organization: An Information Infrastructure Perspective. European Conf. on Information Systems, 92."},{"key":"e_1_3_2_1_42_1","volume-title":"Building Intelligent Systems: A Guide to Machine Learning Engineering","author":"Hulten G.","unstructured":"Hulten, G. 2019. Building Intelligent Systems: A Guide to Machine Learning Engineering. Apress."},{"key":"e_1_3_2_1_43_1","volume-title":"Proc. Int'l Conf. on Software Engineering (ICSE), 1110--1121","author":"Humbatova N.","unstructured":"Humbatova, N. et al. 2020. Taxonomy of real faults in deep learning systems. In Proc. Int'l Conf. on Software Engineering (ICSE), 1110--1121."},{"key":"e_1_3_2_1_44_1","volume-title":"Proc. Int'l Workshop on Conducting Empirical Studies in Industry (CESI) and Software Engineering Research and Industrial Practice (SER&IP), 2--9.","author":"Ishikawa F.","unstructured":"Ishikawa, F. and Yoshioka, N. 2019. How do engineers perceive difficulties in engineering of machine-learning systems? - Questionnaire survey. In Proc. Int'l Workshop on Conducting Empirical Studies in Industry (CESI) and Software Engineering Research and Industrial Practice (SER&IP), 2--9."},{"key":"e_1_3_2_1_45_1","unstructured":"Islam M.J. Nguyen H.A. Pan R. and Rajan H. 2019. What Do Developers Ask About ML Libraries? A Large-scale Study Using Stack Overflow. arXiv 1906.11940."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2012.219"},{"key":"e_1_3_2_1_47_1","first-page":"481","article-title":"Model Assertions for Monitoring and Improving ML Models","volume":"2","author":"Kang D.","year":"2020","unstructured":"Kang, D., Raghavan, D., Bailis, P. and Zaharia, M. 2020. Model Assertions for Monitoring and Improving ML Models. In Proc. of Machine Learning and Systems, 2, 481--496.","journal-title":"Proc. of Machine Learning and Systems"},{"key":"e_1_3_2_1_48_1","volume-title":"Proc. Int'l Conf. Software Engineering: Software Engineering Education and Training (ICSE-SEET), 45--48","author":"K\u00e4stner C.","unstructured":"K\u00e4stner, C. and Kang, E. 2020. Teaching Software Engineering for Al-Enabled Systems. In Proc. Int'l Conf. Software Engineering: Software Engineering Education and Training (ICSE-SEET), 45--48."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2017.2754374"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-020-05872-w"},{"key":"e_1_3_2_1_51_1","unstructured":"Lakshmanan V. Robinson S. and Munn M. 2020. Machine Learning Design Patterns. O'Reilly Media Inc."},{"key":"e_1_3_2_1_52_1","volume-title":"Proc. Workshop on AI Engineering-Software Engineering for AI (WAIN), 133--140","author":"Lewis G.A.","unstructured":"Lewis, G.A., Bellomo, S. and Ozkaya, I. 2021. Characterizing and Detecting Mismatch in Machine-Learning-Enabled Systems. In Proc. Workshop on AI Engineering-Software Engineering for AI (WAIN), 133--140."},{"key":"e_1_3_2_1_53_1","volume-title":"Proc. Int'l Conf. on Software Maintenance and Evolution, 634--638","author":"Lewis G. A.","unstructured":"Lewis, G. A., Ozkaya, I. and Xu X. 2021. Software Architecture Challenges for ML Systems. In Proc. Int'l Conf. on Software Maintenance and Evolution, 634--638."},{"key":"e_1_3_2_1_54_1","volume-title":"Proc. Int'l Workshop on Cooperative and Human Aspects of Software Engineering (CHASE), 2--8.","author":"Li P.L.","unstructured":"Li, P.L., Ko, A.J. and Begel, A. 2017. Cross-Disciplinary Perspectives on Collaborations with Software Engineers. In Proc. Int'l Workshop on Cooperative and Human Aspects of Software Engineering (CHASE), 2--8."},{"key":"e_1_3_2_1_55_1","volume-title":"Proc. Int'l Conf. Agile Software Development, 227--243","author":"Lwakatare L.E.","unstructured":"Lwakatare, L.E., Raj, A., Bosch, J., Olsson, H.H. and Crnkovic, I. 2019. A taxonomy of software engineering challenges for machine learning systems: An empirical investigation. In Proc. Int'l Conf. Agile Software Development, 227--243."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"crossref","unstructured":"Lwakatare L.E. Raj A. Crnkovic I. Bosch J. and Olsson H.H. 2020. Large-scale machine learning systems in real-world industrial settings: A review of challenges and solutions. Information and software technology. 127 106368.","DOI":"10.1016\/j.infsof.2020.106368"},{"key":"e_1_3_2_1_57_1","volume-title":"Proc. Conf. Human Factors in Computing Systems (CHI), 1--14","author":"Madaio M.A.","unstructured":"Madaio, M.A. et al. 2020. Co-Designing Checklists to Understand Organizational Challenges and Opportunities around Fairness in AI. In Proc. Conf. Human Factors in Computing Systems (CHI), 1--14."},{"key":"e_1_3_2_1_58_1","volume-title":"Data Quality: Dimensions, Measurement, Strategy, Management, and Governance","author":"Mahanti R.","year":"2019","unstructured":"Mahanti, R. 2019. Data Quality: Dimensions, Measurement, Strategy, Management, and Governance. Quality Press."},{"key":"e_1_3_2_1_59_1","volume-title":"Proc. Workshop on AI Engineering-Software Engineering for AI (WAIN), 109--112","author":"M\u00e4kinen S.","unstructured":"M\u00e4kinen, S., Skogstr\u00f6m, H., Laaksonen, E. and Mikkonen, T. 2021. Who Needs MLOps: What Data Scientists Seek to Accomplish and How Can MLOps Help? In Proc. Workshop on AI Engineering-Software Engineering for AI (WAIN), 109--112."},{"key":"e_1_3_2_1_60_1","unstructured":"Mart\u00ednez-Fern\u00e1ndez S. Bogner J. Franch X. Oriol M. Siebert J. Trendowicz A. Vollmer A.M. and Wagner S. 2021. Software Engineering for AI-Based Systems: A Survey. arXiv 2105.01984."},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2019.2962680"},{"key":"e_1_3_2_1_62_1","volume-title":"Object-Oriented Software Construction","author":"Meyer B.","unstructured":"Meyer, B. 1997. Object-Oriented Software Construction. Prentice-Hall."},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"crossref","unstructured":"Mistr\u00edk I. Grundy J. van der Hoek A. and Whitehead J. 2010. Collaborative Software Engineering. Springer.","DOI":"10.1007\/978-3-642-10294-3"},{"key":"e_1_3_2_1_64_1","volume-title":"Proc. Conf. Fairness, Accountability, and Transparency, 220--229","author":"Mitchell M.","unstructured":"Mitchell, M., Wu, S., Zaldivar, A., Barnes, P., Vasserman, L., Hutchinson, B., Spitzer, E., Raji, I.D. and Gebru, T. 2019. Model Cards for Model Reporting. In Proc. Conf. Fairness, Accountability, and Transparency, 220--229."},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"crossref","unstructured":"Muiruri D. Lwakatare L. E. K Nurminen J. and Mikkonen T. 2021. Practices and Infrastructures for ML Systems-An Interview Study. TechRxiv 16939192.v1.","DOI":"10.36227\/techrxiv.16939192.v1"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"crossref","unstructured":"Nahar N. Zhou S. Lewis G. and K\u00e4stner C. 2021. Collaboration Challenges in Building ML-Enabled Systems: Communication Documentation Engineering and Process. arXiv 2110.10234.","DOI":"10.1145\/3510003.3510209"},{"key":"e_1_3_2_1_67_1","volume-title":"Proc. Conf. Machine Learning and Systems (MLSys).","author":"O'Leary K.","unstructured":"O'Leary, K. and Uchida, M. 2020. Common problems with creating machine learning pipelines from existing code. In Proc. Conf. Machine Learning and Systems (MLSys)."},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1002\/spip.186"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1109\/MS.2020.2993662","article-title":"What Is Really Different in Engineering AI-Enabled Systems","volume":"37","author":"Ozkaya I.","year":"2020","unstructured":"Ozkaya, I. 2020. What Is Really Different in Engineering AI-Enabled Systems? IEEE Software. 37, 4, 3--6.","journal-title":"IEEE Software."},{"key":"e_1_3_2_1_70_1","volume-title":"Gartner Identifies the Top Strategic Technology Trends for","author":"Panetta K.","year":"2021","unstructured":"Panetta, K. 2020. Gartner Identifies the Top Strategic Technology Trends for 2021. URL: https:\/\/www.gartner.com\/smarterwithgartner\/gartner-top-strategic-technology-trends-for-2021."},{"key":"e_1_3_2_1_71_1","volume-title":"Proc. 26th Int'l Conf. on Intelligent User Interfaces, 585--596","author":"Park S.","unstructured":"Park, S., Wang, A., Kawas, B., Vera Liao, Q., Piorkowski, D. and Danilevsky, M. 2021. Facilitating Knowledge Sharing from Domain Experts to Data Scientists for Building NLP Models. In Proc. 26th Int'l Conf. on Intelligent User Interfaces, 585--596."},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1145\/361598.361623"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1177\/2053951720939605"},{"key":"e_1_3_2_1_74_1","volume-title":"Proc. Conf. Human Factors in Computing Systems (CHI), 667--676","author":"Patel K.","unstructured":"Patel, K., Fogarty, J., Landay, J.A. and Harrison, B. 2008. Investigating statistical machine learning as a tool for software development. In Proc. Conf. Human Factors in Computing Systems (CHI), 667--676."},{"key":"e_1_3_2_1_75_1","volume-title":"Proc. 16th Int'l Conf. on Mining Software Repositories (MSR), 507--517","author":"Pimentel J.F.","unstructured":"Pimentel, J.F., Murta, L., Braganholo, V. and Freire, J. 2019. A large-scale study about quality and reproducibility of Jupyter notebooks. In Proc. 16th Int'l Conf. on Mining Software Repositories (MSR), 507--517."},{"key":"e_1_3_2_1_76_1","volume-title":"et al","author":"Piorkowski D.","year":"2021","unstructured":"Piorkowski, D. et al. 2021. How AI Developers Overcome Communication Challenges in a Multidisciplinary Team: A Case Study. In Proc. ACM on Human-Computer Interaction, 5, (CSCW1), 1--25."},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.1145\/3299887.3299891"},{"key":"e_1_3_2_1_78_1","volume-title":"Proc. Int'l Conf. on Management of Data, 1723--1726","author":"Polyzotis N.","unstructured":"Polyzotis, N., Roy, S., Whang, S.E. and Zinkevich, M. 2017. Data Management Challenges in Production Machine Learning. In Proc. Int'l Conf. on Management of Data, 1723--1726."},{"key":"e_1_3_2_1_79_1","volume-title":"Proc. Machine Learning and Systems, 334--347","author":"Polyzotis N.","unstructured":"Polyzotis, N., Zinkevich, M., Roy, S., Breck, E. and Whang, S. 2019. Data validation for machine learning. In Proc. Machine Learning and Systems, 334--347."},{"key":"e_1_3_2_1_80_1","volume-title":"Proc. Int'l Requirements Engineering Conf. Workshops (REW), 241--244","author":"Rahimi M.","unstructured":"Rahimi, M., Guo, J.L.C., Kokaly, S. and Chechik, M. 2019. Toward Requirements Specification for Machine-Learned Components. In Proc. Int'l Requirements Engineering Conf. Workshops (REW), 241--244."},{"key":"e_1_3_2_1_81_1","volume-title":"Reality: Practitioner Perspectives on Enablers for Shifting Organizational Practices. Proc. ACM Hum.-Comput. Interact. 5, CSCW1, 1--23.","author":"Rakova B.","year":"2021","unstructured":"Rakova, B., Yang, J., Cramer, H. and Chowdhury, R. 2021. Where Responsible AI meets Reality: Practitioner Perspectives on Enablers for Shifting Organizational Practices. Proc. ACM Hum.-Comput. Interact. 5, CSCW1, 1--23."},{"key":"e_1_3_2_1_82_1","volume-title":"Overton: A data system for monitoring and improving machine-learned products. arXiv","author":"R\u00e9 C.","year":"2019","unstructured":"R\u00e9, C., Niu, F., Gudipati, P. and Srisuwananukorn, C. 2019. Overton: A data system for monitoring and improving machine-learned products. arXiv 1909.05372."},{"key":"e_1_3_2_1_83_1","doi-asserted-by":"crossref","unstructured":"Salay R. Queiroz R. and Czarnecki K. 2017. An Analysis of ISO 26262: Using Machine Learning Safely in Automotive Software. arXiv 1709.02435.","DOI":"10.4271\/2018-01-1075"},{"key":"e_1_3_2_1_84_1","volume-title":"Proc. Conf. on Human Factors in Computing Systems (CHI). 1--15","author":"Sambasivan N.","unstructured":"Sambasivan, N. et al. 2021. \"Everyone wants to do the model work, not the data work\": Data Cascades in High-Stakes AI. In Proc. Conf. on Human Factors in Computing Systems (CHI). 1--15."},{"key":"e_1_3_2_1_85_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2011.64"},{"key":"e_1_3_2_1_86_1","doi-asserted-by":"publisher","DOI":"10.14778\/3229863.3229867"},{"key":"e_1_3_2_1_87_1","volume-title":"et al","author":"Sculley D.","year":"2015","unstructured":"Sculley, D. et al. 2015. Hidden Technical Debt in Machine Learning Systems. Advances in Neural Information Processing Systems 28. 2503--2511."},{"key":"e_1_3_2_1_88_1","volume-title":"Proc. Int'l Conf. Knowledge Discovery and Data Mining, 274--282","author":"Sculley D.","unstructured":"Sculley, D., Otey, M.E., Pohl, M., Spitznagel, B., Hainsworth, J. and Zhou, Y. 2011. Detecting adversarial advertisements in the wild. In Proc. Int'l Conf. Knowledge Discovery and Data Mining, 274--282."},{"key":"e_1_3_2_1_89_1","volume-title":"et al","author":"Sendak M.P.","year":"2020","unstructured":"Sendak, M.P. et al. 2020. Real-World Integration of a Sepsis Deep Learning Technology Into Routine Clinical Care: Implementation Study. JMIR medical informatics. 8, 7, e15182."},{"key":"e_1_3_2_1_90_1","volume-title":"Proc. Int'l Symposium on Empirical Software Engineering and Measurement, 1--12","author":"Serban A.","unstructured":"Serban, A., van der Blom, K., Hoos, H. and Visser, J. 2020. Adoption and Effects of Software Engineering Best Practices in Machine Learning. In Proc. Int'l Symposium on Empirical Software Engineering and Measurement, 1--12."},{"key":"e_1_3_2_1_91_1","volume-title":"Proc. IFIP Working Conf. Human Work Interaction Design, 200--214","author":"Seymoens T.","unstructured":"Seymoens, T., Ongenae, F. and Jacobs, A. 2018. A methodology to involve domain experts and machine learning techniques in the design of human-centered algorithms. In Proc. IFIP Working Conf. Human Work Interaction Design, 200--214."},{"key":"e_1_3_2_1_92_1","doi-asserted-by":"publisher","DOI":"10.1145\/3419764"},{"key":"e_1_3_2_1_93_1","volume-title":"Proc. Int'l Conf. on the Quality of Information and Communications Technology, 17--31","author":"Siebert J.","unstructured":"Siebert, J., Joeckel, L., Heidrich, J., Nakamichi, K., Ohashi, K., Namba, I., Yamamoto, R. and Aoyama, M. 2020. Towards Guidelines for Assessing Qualities of Machine Learning Systems. In Proc. Int'l Conf. on the Quality of Information and Communications Technology, 17--31."},{"key":"e_1_3_2_1_94_1","volume-title":"Proc. ACM Program. Lang. 3, POPL, 1--30","author":"Singh G.","unstructured":"Singh, G., Gehr, T., P\u00fcschel, M. and Vechev, M. 2019. An abstract domain for certifying neural networks. Proc. ACM Program. Lang. 3, POPL, 1--30."},{"key":"e_1_3_2_1_95_1","doi-asserted-by":"publisher","DOI":"10.22215\/timreview\/758"},{"key":"e_1_3_2_1_96_1","volume-title":"Proc. Int'l Symposium on Empirical Software Engineering and Measurement (ESEM), 1--6.","author":"Nascimento E.","unstructured":"d. S. Nascimento, E. et al. 2019. Understanding Development Process of Machine Learning Systems: Challenges and Solutions. In Proc. Int'l Symposium on Empirical Software Engineering and Measurement (ESEM), 1--6."},{"key":"e_1_3_2_1_97_1","volume-title":"Proc. Int'l Conf. Software Engineering (ICSE), 241--250","author":"de Souza C.R.B.","unstructured":"de Souza, C.R.B. and Redmiles, D.F. 2008. An Empirical Study of Software Developers' Management of Dependencies and Changes. In Proc. Int'l Conf. Software Engineering (ICSE), 241--250."},{"key":"e_1_3_2_1_98_1","unstructured":"Strauss A. and Corbin J. 1994. Grounded theory methodology: An overview. Handbook of qualitative research. N.K. Denzin ed. 273--285."},{"key":"e_1_3_2_1_99_1","volume-title":"Basics of Qualitative Research: Grounded Theory Procedures and Techniques","author":"Strauss A.","unstructured":"Strauss, A. and Corbin, J.M. Basics of Qualitative Research: Grounded Theory Procedures and Techniques. SAGE Publications."},{"key":"e_1_3_2_1_100_1","doi-asserted-by":"publisher","DOI":"10.3390\/make3020020"},{"key":"e_1_3_2_1_101_1","volume-title":"et al","author":"Tram\u00e8r F.","year":"2017","unstructured":"Tram\u00e8r, F. et al. 2017. FairTest: Discovering Unwarranted Associations in Data-Driven Applications. In Proc. European Symposium on Security and Privacy (EuroSP), 401--416."},{"key":"e_1_3_2_1_102_1","first-page":"12","article-title":"The T-Shaped Engineer","volume":"30","author":"Tranquillo J.","year":"2017","unstructured":"Tranquillo, J. 2017. The T-Shaped Engineer. Journal of Engineering Education Transformations. 30, 4, 12--24.","journal-title":"Journal of Engineering Education Transformations."},{"key":"e_1_3_2_1_103_1","volume-title":"Proc. Int'l Requirements Engineering Conf. Workshops (REW), 245--251","author":"Vogelsang A.","unstructured":"Vogelsang, A. and Borg, M. 2019. Requirements Engineering for Machine Learning: Perspectives from Data Scientists. In Proc. Int'l Requirements Engineering Conf. Workshops (REW), 245--251."},{"key":"e_1_3_2_1_104_1","unstructured":"Wagstaff K. 2012. Machine Learning that Matters. arXiv 1206.4656."},{"key":"e_1_3_2_1_105_1","volume-title":"Proc. Human-Computer Interaction. 3, CSCW, 39","author":"Wang A.Y.","unstructured":"Wang, A.Y., Mittal, A., Brooks, C. and Oney, S. 2019. How Data Scientists Use Computational Notebooks for Real-Time Collaboration. Proc. Human-Computer Interaction. 3, CSCW, 39."},{"issue":"9","key":"e_1_3_2_1_106_1","first-page":"1857","article-title":"How does Machine Learning Change Software Development Practices","volume":"47","author":"Wan Z.","year":"2019","unstructured":"Wan, Z., Xia, X., Lo, D. and Murphy, G.C. 2019. How does Machine Learning Change Software Development Practices? IEEE Transactions on Software Engineering, 47(9), 1857--1871.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_1_107_1","volume-title":"Proc. Int'l Conf. Software Engineering, 347--357","author":"Waterman M.","unstructured":"Waterman, M., Noble, J. and Allan, G. 2015. How Much Up-Front? A Grounded theory of Agile Architecture. In Proc. Int'l Conf. Software Engineering, 347--357."},{"key":"e_1_3_2_1_108_1","unstructured":"Staff V. B. 2019. Why do 87% of data science projects never make it into production? URL: https:\/\/venturebeat.com\/2019\/07\/19\/why-do-87-of-data-science-projects-never-make-it-into-production\/."},{"key":"e_1_3_2_1_109_1","doi-asserted-by":"crossref","unstructured":"Wiens J. et al. 2019. Do no harm: A roadmap for responsible machine learning for health care. Nature medicine. 25 9 1337--1340.","DOI":"10.1038\/s41591-019-0548-6"},{"key":"e_1_3_2_1_110_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2010.11.920"},{"key":"e_1_3_2_1_111_1","volume-title":"Proc. Conf. Designing Interactive Systems, 573--584","author":"Yang Q.","unstructured":"Yang, Q., Suh, J., Chen, N.-C. and Ramos, G. 2018. Grounding Interactive Machine Learning Tool Design in How Non-Experts Actually Build Models. In Proc. Conf. Designing Interactive Systems, 573--584."},{"key":"e_1_3_2_1_112_1","volume-title":"Proc. AAAI Spring Symposium Series, 406--411","author":"Yang Q.","unstructured":"Yang, Q. The role of design in creating machine-learning-enhanced user experience. In Proc. AAAI Spring Symposium Series, 406--411."},{"key":"e_1_3_2_1_113_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSA-C.2019.00055"},{"key":"e_1_3_2_1_114_1","volume-title":"Proc. Human-Computer Interaction. 4, CSCW1, 1--23","author":"Zhang A.X.","unstructured":"Zhang, A.X., Muller, M. and Wang, D. 2020. How do data science workers collaborate? Roles, workflows, and tools. Proc. Human-Computer Interaction. 4, CSCW1, 1--23."},{"key":"e_1_3_2_1_115_1","volume-title":"Proc. Int'l Conf. Software Engineering (ICSE), 445--456","author":"Zhou S.","unstructured":"Zhou, S., Vasilescu, B. and K\u00e4stner, C. 2020. How Has Forking Changed in the Last 20 Years? A Study of Hard Forks on GitHub. In Proc. Int'l Conf. Software Engineering (ICSE), 445--456."},{"key":"e_1_3_2_1_116_1","unstructured":"Zinkevich M. 2017. Rules of machine learning: Best practices for ML engineering. URL: https:\/\/developers.google.com\/machine-learning\/guides\/rules-of-ml."}],"event":{"name":"ICSE '22: 44th International Conference on Software Engineering","location":"Pittsburgh Pennsylvania","acronym":"ICSE '22","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 44th International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3510003.3510209","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3510003.3510209","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3510003.3510209","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:12:24Z","timestamp":1750191144000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3510003.3510209"}},"subtitle":["communication, documentation, engineering, and process"],"short-title":[],"issued":{"date-parts":[[2022,5,21]]},"references-count":116,"alternative-id":["10.1145\/3510003.3510209","10.1145\/3510003"],"URL":"https:\/\/doi.org\/10.1145\/3510003.3510209","relation":{},"subject":[],"published":{"date-parts":[[2022,5,21]]},"assertion":[{"value":"2022-07-05","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}