{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,18]],"date-time":"2025-12-18T14:25:52Z","timestamp":1766067952576,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":29,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,14]],"date-time":"2024-04-14T00:00:00Z","timestamp":1713052800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,14]]},"DOI":"10.1145\/3639476.3639771","type":"proceedings-article","created":{"date-parts":[[2024,5,24]],"date-time":"2024-05-24T15:15:01Z","timestamp":1716563701000},"page":"1-5","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Expert Monitoring: Human-Centered Concept Drift Detection in Machine Learning Operations"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-1335-6559","authenticated-orcid":false,"given":"Joran","family":"Leest","sequence":"first","affiliation":[{"name":"Vrije Universiteit Amsterdam, Amsterdam, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7194-3159","authenticated-orcid":false,"given":"Claudia","family":"Raibulet","sequence":"additional","affiliation":[{"name":"Vrije Universiteit Amsterdam, Amsterdam, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9333-7101","authenticated-orcid":false,"given":"Ilias","family":"Gerostathopoulos","sequence":"additional","affiliation":[{"name":"Vrije Universiteit Amsterdam, Amsterdam, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2234-0845","authenticated-orcid":false,"given":"Patricia","family":"Lago","sequence":"additional","affiliation":[{"name":"Vrije Universiteit Amsterdam, Amsterdam, The Netherlands"}]}],"member":"320","published-online":{"date-parts":[[2024,5,24]]},"reference":[{"doi-asserted-by":"publisher","key":"e_1_3_2_1_1_1","DOI":"10.1109\/ACCESS.2020.3042657"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_2_1","DOI":"10.1109\/BigData.2017.8258038"},{"key":"e_1_3_2_1_3_1","volume-title":"International conference on machine learning. PMLR, 1617--1629","author":"Chen Mayee","year":"2021","unstructured":"Mayee Chen, Karan Goel, Nimit S Sohoni, Fait Poms, Kayvon Fatahalian, and Christopher R\u00e9. 2021. Mandoline: Model evaluation under distribution shift. In International conference on machine learning. PMLR, 1617--1629."},{"key":"e_1_3_2_1_4_1","volume-title":"Markov chain Monte Carlo methods: computation and inference. Handbook of econometrics 5","author":"Chib Siddhartha","year":"2001","unstructured":"Siddhartha Chib. 2001. Markov chain Monte Carlo methods: computation and inference. Handbook of econometrics 5 (2001), 3569--3649."},{"key":"e_1_3_2_1_5_1","volume-title":"International Conference on Machine Learning. PMLR, 4087--4111","author":"Cobb Oliver","year":"2022","unstructured":"Oliver Cobb and Arnaud Van Looveren. 2022. Context-aware drift detection. In International Conference on Machine Learning. PMLR, 4087--4111."},{"key":"e_1_3_2_1_6_1","volume-title":"A survey on concept drift adaptation. ACM computing surveys (CSUR) 46, 4","author":"Gama Jo\u00e3o","year":"2014","unstructured":"Jo\u00e3o Gama, Indr\u0117 \u017dliobait\u0117, Albert Bifet, Mykola Pechenizkiy, and Abdelhamid Bouchachia. 2014. A survey on concept drift adaptation. ACM computing surveys (CSUR) 46, 4 (2014), 1--37."},{"key":"e_1_3_2_1_7_1","volume-title":"Leveraging unlabeled data to predict out-of-distribution performance. arXiv preprint arXiv:2201.04234","author":"Garg Saurabh","year":"2022","unstructured":"Saurabh Garg, Sivaraman Balakrishnan, Zachary C Lipton, Behnam Neyshabur, and Hanie Sedghi. 2022. Leveraging unlabeled data to predict out-of-distribution performance. arXiv preprint arXiv:2201.04234 (2022)."},{"volume-title":"Bayesian methods: A social and behavioral sciences approach","author":"Gill Jeff","unstructured":"Jeff Gill. 2014. Bayesian methods: A social and behavioral sciences approach. Vol. 20. CRC press.","key":"e_1_3_2_1_8_1"},{"key":"e_1_3_2_1_9_1","volume-title":"Identifying the Context Shift between Test Benchmarks and Production Data. arXiv preprint arXiv:2207.01059","author":"Groh Matthew","year":"2022","unstructured":"Matthew Groh. 2022. Identifying the Context Shift between Test Benchmarks and Production Data. arXiv preprint arXiv:2207.01059 (2022)."},{"key":"e_1_3_2_1_10_1","volume-title":"Proceedings of the IEEE\/CVF international conference on computer vision. 1134--1144","author":"Guillory Devin","year":"2021","unstructured":"Devin Guillory, Vaishaal Shankar, Sayna Ebrahimi, Trevor Darrell, and Ludwig Schmidt. 2021. Predicting with confidence on unseen distributions. In Proceedings of the IEEE\/CVF international conference on computer vision. 1134--1144."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_11_1","DOI":"10.1111\/coin.12520"},{"volume-title":"Designing machine learning systems. \"O'Reilly Media","author":"Huyen Chip","unstructured":"Chip Huyen. 2022. Designing machine learning systems. \"O'Reilly Media, Inc.\".","key":"e_1_3_2_1_12_1"},{"key":"e_1_3_2_1_13_1","volume-title":"Machine learning operations (mlops): Overview, definition, and architecture","author":"Kreuzberger Dominik","year":"2023","unstructured":"Dominik Kreuzberger, Niklas K\u00fchl, and Sebastian Hirschl. 2023. Machine learning operations (mlops): Overview, definition, and architecture. IEEE Access (2023)."},{"unstructured":"Cloudera Fast Forward Labs. 2021. Inferring Concept Drift Without Labeled Data. (2021).","key":"e_1_3_2_1_14_1"},{"key":"e_1_3_2_1_15_1","volume-title":"Evolvability of Machine Learning-based Systems: An Architectural Design Decision Framework. In 2023 IEEE 20th International Conference on Software Architecture Companion (ICSA-C). IEEE, 106--110","author":"Leest Joran","year":"2023","unstructured":"Joran Leest, Ilias Gerostathopoulos, and Claudia Raibulet. 2023. Evolvability of Machine Learning-based Systems: An Architectural Design Decision Framework. In 2023 IEEE 20th International Conference on Software Architecture Companion (ICSA-C). IEEE, 106--110."},{"key":"e_1_3_2_1_16_1","volume-title":"Handling Concept Drift in Global Time Series Forecasting. arXiv preprint arXiv:2304.01512","author":"Liu Ziyi","year":"2023","unstructured":"Ziyi Liu, Rakshitha Godahewa, Kasun Bandara, and Christoph Bergmeir. 2023. Handling Concept Drift in Global Time Series Forecasting. arXiv preprint arXiv:2304.01512 (2023)."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_17_1","DOI":"10.1109\/TKDE.2018.2876857"},{"key":"e_1_3_2_1_18_1","first-page":"77","article-title":"Matchmaker: Data drift mitigation in machine learning for large-scale systems","volume":"4","author":"Mallick Ankur","year":"2022","unstructured":"Ankur Mallick, Kevin Hsieh, Behnaz Arzani, and Gauri Joshi. 2022. Matchmaker: Data drift mitigation in machine learning for large-scale systems. Proceedings of Machine Learning and Systems 4 (2022), 77--94.","journal-title":"Proceedings of Machine Learning and Systems"},{"key":"e_1_3_2_1_19_1","volume-title":"Conjugate Bayesian analysis of the Gaussian distribution. def 1, 2\u03c32","author":"Murphy Kevin P","year":"2007","unstructured":"Kevin P Murphy. 2007. Conjugate Bayesian analysis of the Gaussian distribution. def 1, 2\u03c32 (2007), 16."},{"key":"e_1_3_2_1_20_1","first-page":"24","article-title":"From Cleaning before ML to Cleaning for ML","volume":"44","author":"Neutatz Felix","year":"2021","unstructured":"Felix Neutatz, Binger Chen, Ziawasch Abedjan, and Eugene Wu. 2021. From Cleaning before ML to Cleaning for ML. IEEE Data Eng. Bull. 44, 1 (2021), 24--41.","journal-title":"IEEE Data Eng. Bull."},{"key":"e_1_3_2_1_21_1","volume-title":"M\u00e1rio AT Figueiredo, and Pedro Bizarro","author":"Pombal Jos\u00e9","year":"2022","unstructured":"Jos\u00e9 Pombal, Andr\u00e9 F Cruz, Jo\u00e3o Bravo, Pedro Saleiro, M\u00e1rio AT Figueiredo, and Pedro Bizarro. 2022. Understanding Unfairness in Fraud Detection through Model and Data Bias Interactions. arXiv preprint arXiv:2207.06273 (2022)."},{"key":"e_1_3_2_1_22_1","volume-title":"Failing loudly: An empirical study of methods for detecting dataset shift. Advances in Neural Information Processing Systems 32","author":"Rabanser Stephan","year":"2019","unstructured":"Stephan Rabanser, Stephan G\u00fcnnemann, and Zachary Lipton. 2019. Failing loudly: An empirical study of methods for detecting dataset shift. Advances in Neural Information Processing Systems 32 (2019)."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_23_1","DOI":"10.1016\/j.eswa.2017.04.008"},{"key":"e_1_3_2_1_24_1","volume-title":"J Wilson, and S Sharples.","author":"Shadbolt Nigel R","year":"2015","unstructured":"Nigel R Shadbolt, Paul R Smart, J Wilson, and S Sharples. 2015. Knowledge elicitation. Evaluation of human work (2015), 163--200."},{"key":"e_1_3_2_1_25_1","volume-title":"Operationalizing machine learning: An interview study. arXiv preprint arXiv:2209.09125","author":"Shankar Shreya","year":"2022","unstructured":"Shreya Shankar, Rolando Garcia, Joseph M Hellerstein, and Aditya G Parameswaran. 2022. Operationalizing machine learning: An interview study. arXiv preprint arXiv:2209.09125 (2022)."},{"key":"e_1_3_2_1_26_1","volume-title":"Towards observability for production machine learning pipelines. arXiv preprint arXiv:2108.13557","author":"Shankar Shreya","year":"2021","unstructured":"Shreya Shankar and Aditya Parameswaran. 2021. Towards observability for production machine learning pipelines. arXiv preprint arXiv:2108.13557 (2021)."},{"key":"e_1_3_2_1_27_1","volume-title":"A Human-Centric Take on Model Monitoring. arXiv preprint arXiv:2206.02868","author":"Shergadwala Murtuza N","year":"2022","unstructured":"Murtuza N Shergadwala, Himabindu Lakkaraju, and Krishnaram Kenthapadi. 2022. A Human-Centric Take on Model Monitoring. arXiv preprint arXiv:2206.02868 (2022)."},{"key":"e_1_3_2_1_28_1","volume-title":"Software engineering (ed.)","author":"Sommerville Ian","year":"2011","unstructured":"Ian Sommerville. 2011. Software engineering (ed.). America: Pearson Education Inc (2011)."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_29_1","DOI":"10.1007\/s10618-015-0448-4"}],"event":{"sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"],"acronym":"ICSE-NIER'24","name":"ICSE-NIER'24: 2024 ACM\/IEEE 44th International Conference on Software Engineering: New Ideas and Emerging Results","location":"Lisbon Portugal"},"container-title":["Proceedings of the 2024 ACM\/IEEE 44th International Conference on Software Engineering: New Ideas and Emerging Results"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3639476.3639771","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3639476.3639771","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T22:53:39Z","timestamp":1750287219000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3639476.3639771"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,14]]},"references-count":29,"alternative-id":["10.1145\/3639476.3639771","10.1145\/3639476"],"URL":"https:\/\/doi.org\/10.1145\/3639476.3639771","relation":{},"subject":[],"published":{"date-parts":[[2024,4,14]]},"assertion":[{"value":"2024-05-24","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}