{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,18]],"date-time":"2025-12-18T10:26:45Z","timestamp":1766053605466,"version":"3.48.0"},"reference-count":158,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"12","license":[{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IIEEE Trans. Software Eng."],"published-print":{"date-parts":[[2025,12]]},"DOI":"10.1109\/tse.2025.3602520","type":"journal-article","created":{"date-parts":[[2025,8,26]],"date-time":"2025-08-26T19:21:36Z","timestamp":1756236096000},"page":"3218-3246","source":"Crossref","is-referenced-by-count":0,"title":["From Tea Leaves to System Maps: A Survey and Framework on Context-Aware Machine Learning Monitoring"],"prefix":"10.1109","volume":"51","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-1335-6559","authenticated-orcid":false,"given":"Joran","family":"Leest","sequence":"first","affiliation":[{"name":"Department of Computer Science, Faculty of Sciences, Vrije Universiteit Amsterdam, Amsterdam, HV, The Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7194-3159","authenticated-orcid":false,"given":"Claudia","family":"Raibulet","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Faculty of Sciences, Vrije Universiteit Amsterdam, Amsterdam, HV, The Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2234-0845","authenticated-orcid":false,"given":"Patricia","family":"Lago","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Faculty of Sciences, Vrije Universiteit Amsterdam, Amsterdam, HV, The Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9333-7101","authenticated-orcid":false,"given":"Ilias","family":"Gerostathopoulos","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Faculty of Sciences, Vrije Universiteit Amsterdam, Amsterdam, HV, The Netherlands"}]}],"member":"263","reference":[{"doi-asserted-by":"publisher","key":"ref1","DOI":"10.1038\/s41598-022-15245-z"},{"doi-asserted-by":"publisher","key":"ref2","DOI":"10.1145\/3653697"},{"doi-asserted-by":"publisher","key":"ref3","DOI":"10.1145\/3240323.3240370"},{"doi-asserted-by":"publisher","key":"ref4","DOI":"10.1109\/BigData.2017.8258038"},{"doi-asserted-by":"publisher","key":"ref5","DOI":"10.1145\/2523813"},{"doi-asserted-by":"publisher","key":"ref6","DOI":"10.1109\/TKDE.2018.2876857"},{"doi-asserted-by":"publisher","key":"ref7","DOI":"10.1002\/widm.1327"},{"year":"2022","author":"Garg","article-title":"Leveraging unlabeled data to predict out-of-distribution performance","key":"ref8"},{"doi-asserted-by":"publisher","key":"ref9","DOI":"10.1109\/ICCV48922.2021.00117"},{"key":"ref10","first-page":"19274","article-title":"Agreement-on-the-line: Predicting the performance of neural networks under distribution shift","volume":"35","author":"Baek","year":"2022","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref11","first-page":"1","article-title":"Data validation for machine learning","volume-title":"Proc. MLSys","author":"Breck","year":"2019"},{"doi-asserted-by":"publisher","key":"ref12","DOI":"10.14778\/3229863.3229867"},{"doi-asserted-by":"publisher","key":"ref13","DOI":"10.1145\/3583780.3614786"},{"key":"ref14","first-page":"61","article-title":"Automating data quality validation for dynamic data ingestion","author":"Redyuk","year":"2021"},{"doi-asserted-by":"publisher","key":"ref15","DOI":"10.1609\/hcomp.v10i1.21997"},{"key":"ref16","first-page":"1","article-title":"Hidden technical debt in machine learning systems","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"28","author":"Sculley","year":"2015"},{"doi-asserted-by":"publisher","key":"ref17","DOI":"10.1145\/3292500.3330744"},{"doi-asserted-by":"publisher","key":"ref18","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"doi-asserted-by":"publisher","key":"ref19","DOI":"10.1145\/3533378"},{"doi-asserted-by":"publisher","key":"ref20","DOI":"10.1109\/ICSE-SEIP52600.2021.00034"},{"doi-asserted-by":"publisher","key":"ref21","DOI":"10.1145\/3510457.3513058"},{"doi-asserted-by":"publisher","key":"ref22","DOI":"10.1109\/MC.2022.3161161"},{"volume-title":"Designing Machine Learning Systems","year":"2022","author":"Huyen","key":"ref23"},{"doi-asserted-by":"publisher","key":"ref24","DOI":"10.1145\/3318464.3384707"},{"key":"ref25","first-page":"1","article-title":"Overton: A data system for monitoring and improving machine-learned products","volume-title":"Proc. 10th Annu. Conf. Innovative Data Syst. Res. (CIDR)","author":"R\u00e9","year":"2020"},{"key":"ref26","first-page":"1","article-title":"Why did the distribution change?","volume-title":"Proc. Mach. Learn. Res.","author":"Kailash","year":"2021"},{"key":"ref27","first-page":"1","article-title":"Diagnosing failures of fairness transfer across distribution shift in real-world medical settings","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"35","author":"Schrouff","year":"2022"},{"doi-asserted-by":"publisher","key":"ref28","DOI":"10.1111\/exsy.12059"},{"doi-asserted-by":"publisher","key":"ref29","DOI":"10.1145\/3639476.3639771"},{"doi-asserted-by":"publisher","key":"ref30","DOI":"10.1016\/j.dsm.2022.07.004"},{"year":"2021","article-title":"Inferring concept drift without labeled data","key":"ref31"},{"doi-asserted-by":"publisher","key":"ref32","DOI":"10.1007\/s10618-015-0448-4"},{"doi-asserted-by":"publisher","key":"ref33","DOI":"10.1145\/3522664.3528590"},{"doi-asserted-by":"publisher","key":"ref34","DOI":"10.14778\/3565838.3565853"},{"doi-asserted-by":"publisher","key":"ref35","DOI":"10.1145\/3514094.3534157"},{"doi-asserted-by":"publisher","key":"ref36","DOI":"10.1111\/coin.12520"},{"key":"ref37","first-page":"1","article-title":"Labelless concept drift detection and explanation","volume-title":"Proc. Workshop Robust AI Financial Services, Data, Fairness, Explainability, Trustworthiness, Privacy (NeurIPS)","author":"Zheng","year":"2019"},{"doi-asserted-by":"publisher","key":"ref38","DOI":"10.1007\/978-3-030-27615-7_17"},{"doi-asserted-by":"publisher","key":"ref39","DOI":"10.1109\/MC.2021.3057686"},{"key":"ref40","doi-asserted-by":"crossref","first-page":"9845","DOI":"10.1007\/978-981-16-8193-6","article-title":"Sequential covariate shift detection using classifier two-sample tests","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Jang","year":"2022"},{"doi-asserted-by":"publisher","key":"ref41","DOI":"10.1109\/VIS49827.2021.9623289"},{"key":"ref42","article-title":"Monitoring bias and fairness in machine learning models: A review","author":"Bacelar","year":"2021","journal-title":"ScienceOpen Preprints"},{"doi-asserted-by":"publisher","key":"ref43","DOI":"10.1145\/3442188.3445865"},{"key":"ref44","first-page":"1617","article-title":"Mandoline: Model evaluation under distribution shift","volume-title":"Int. Conf. Mach. Learn.","author":"Chen","year":"2021"},{"doi-asserted-by":"publisher","key":"ref45","DOI":"10.1007\/978-3-030-26250-1_29"},{"doi-asserted-by":"publisher","key":"ref46","DOI":"10.1007\/978-3-031-05434-1_1"},{"doi-asserted-by":"publisher","key":"ref47","DOI":"10.1016\/j.jss.2021.111096"},{"key":"ref48","first-page":"160","article-title":"Runaway feedback loops in predictive policing","volume-title":"Proc. Conf. Fairness, Accountability Transparency","author":"Ensign","year":"2018"},{"doi-asserted-by":"publisher","key":"ref49","DOI":"10.3991\/ijim.v17i23.43479"},{"doi-asserted-by":"publisher","key":"ref50","DOI":"10.1109\/OCIT59427.2023.10431343"},{"doi-asserted-by":"publisher","key":"ref51","DOI":"10.1109\/ACCESS.2023.3262138"},{"doi-asserted-by":"publisher","key":"ref52","DOI":"10.1109\/TSE.2019.2962027"},{"year":"2023","author":"Chandrasekaran","article-title":"Test & evaluation best practices for machine learning-enabled systems","key":"ref53"},{"doi-asserted-by":"publisher","key":"ref54","DOI":"10.1109\/SaTML54575.2023.00038"},{"doi-asserted-by":"publisher","key":"ref55","DOI":"10.1016\/j.patter.2023.100780"},{"key":"ref56","first-page":"1","article-title":"Context: The missing piece in the machine learning lifecycle","volume-title":"Proc. Common Model Infrastructure Workshop","author":"Garcia","year":"2018"},{"doi-asserted-by":"publisher","key":"ref57","DOI":"10.1038\/s41746-020-0262-2"},{"doi-asserted-by":"publisher","key":"ref58","DOI":"10.1207\/s15327051hci16234_02"},{"doi-asserted-by":"publisher","key":"ref59","DOI":"10.1504\/IJAHUC.2007.014070"},{"doi-asserted-by":"publisher","key":"ref60","DOI":"10.5220\/0010818200003119"},{"doi-asserted-by":"publisher","key":"ref61","DOI":"10.1109\/ICSA.2017.26"},{"doi-asserted-by":"publisher","key":"ref62","DOI":"10.1016\/j.infsof.2019.04.011"},{"doi-asserted-by":"publisher","key":"ref63","DOI":"10.1016\/j.datak.2017.03.008"},{"doi-asserted-by":"publisher","key":"ref64","DOI":"10.1007\/978-3-319-24465-5_7"},{"doi-asserted-by":"publisher","key":"ref65","DOI":"10.3233\/IDA-194515"},{"doi-asserted-by":"publisher","key":"ref66","DOI":"10.1109\/CVPR.2013.419"},{"doi-asserted-by":"publisher","key":"ref67","DOI":"10.1145\/3661826"},{"key":"ref68","first-page":"587","article-title":"Designing monitoring strategies for deployed machine learning algorithms: Navigating performativity through a causal lens","volume-title":"Proc. Causal Learn. Reasoning","author":"Feng","year":"2024"},{"doi-asserted-by":"publisher","key":"ref69","DOI":"10.1109\/CDC49753.2023.10383566"},{"key":"ref70","first-page":"4087","article-title":"Context-aware drift detection","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Cobb","year":"2022"},{"doi-asserted-by":"publisher","key":"ref71","DOI":"10.1145\/3526073.3527590"},{"doi-asserted-by":"publisher","key":"ref72","DOI":"10.1109\/CAIN58948.2023.00009"},{"doi-asserted-by":"publisher","key":"ref73","DOI":"10.1145\/3531232.3531261"},{"doi-asserted-by":"publisher","key":"ref74","DOI":"10.32473\/flairs.36.133240"},{"doi-asserted-by":"publisher","key":"ref75","DOI":"10.1145\/3578356.3592593"},{"doi-asserted-by":"publisher","key":"ref76","DOI":"10.1007\/978-3-030-20485-3_17"},{"doi-asserted-by":"publisher","key":"ref77","DOI":"10.1109\/TNNLS.2013.2271915"},{"doi-asserted-by":"publisher","key":"ref78","DOI":"10.1109\/DeepTest52559.2021.00007"},{"key":"ref79","first-page":"1","article-title":"Why did the model fail?\u2019: Attributing model performance changes to distribution shifts","volume-title":"Proc. Mach. Learn. Res.","author":"Zhang","year":"2023"},{"doi-asserted-by":"publisher","key":"ref80","DOI":"10.1145\/3076246.3076251"},{"doi-asserted-by":"publisher","key":"ref81","DOI":"10.1145\/3626772.3657834"},{"year":"2025","article-title":"The fastest and most powerful platform for building AI products","key":"ref82"},{"doi-asserted-by":"publisher","key":"ref83","DOI":"10.18653\/v1\/2024.findings-acl.137"},{"issue":"2","key":"ref84","first-page":"1","article-title":"Miss intensified customer churn prediction: Connectivity with weighted multi-layer perceptron and enhanced multipath back propagation: English","volume":"4","author":"Panimalar","year":"2025","journal-title":"Int. J. Adv. Sci. Computer Appl."},{"doi-asserted-by":"publisher","key":"ref85","DOI":"10.1007\/s42979-024-02722-7"},{"year":"2025","author":"Blog","article-title":"End-to-end blueprint for customer churn modeling and prediction \u2013 Part 1","key":"ref86"},{"year":"2025","author":"Analytics","article-title":"Data pipeline architecture for churn prediction: Automating data flow with aws services","key":"ref87"},{"year":"2025","author":"Developers","article-title":"Churn prediction for game developers using google analytics 4 (ga4) and bigquery ML","key":"ref88"},{"year":"2025","author":"Services","article-title":"Subscriber churn prediction and retention on aws","key":"ref89"},{"doi-asserted-by":"publisher","key":"ref90","DOI":"10.1109\/ICDM.2016.0087"},{"doi-asserted-by":"publisher","key":"ref91","DOI":"10.1145\/3617694.3623239"},{"doi-asserted-by":"publisher","key":"ref92","DOI":"10.1145\/3399579.3399868"},{"doi-asserted-by":"publisher","key":"ref93","DOI":"10.1007\/978-3-031-29786-1_16"},{"doi-asserted-by":"publisher","key":"ref94","DOI":"10.1145\/3593013.3594028"},{"doi-asserted-by":"publisher","key":"ref95","DOI":"10.1145\/3366423.3380306"},{"key":"ref96","first-page":"1","article-title":"On the (in)feasibility of ml backdoor detection as an hypothesis testing problem","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Pichler","year":"2024"},{"doi-asserted-by":"publisher","key":"ref97","DOI":"10.1109\/ICDE48307.2020.00140"},{"doi-asserted-by":"publisher","key":"ref98","DOI":"10.1007\/978-3-031-44267-4_15"},{"doi-asserted-by":"publisher","key":"ref99","DOI":"10.1016\/j.clsr.2024.105986"},{"doi-asserted-by":"publisher","key":"ref100","DOI":"10.37934\/araset.40.2.127139"},{"doi-asserted-by":"publisher","key":"ref101","DOI":"10.1007\/s10618-022-00845-0"},{"key":"ref102","first-page":"1","article-title":"Estimating and explaining model performance when both covariates and labels shift","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"35","author":"Chen","year":"2022"},{"doi-asserted-by":"publisher","key":"ref103","DOI":"10.1109\/ICTAI56018.2022.00009"},{"doi-asserted-by":"publisher","key":"ref104","DOI":"10.1016\/j.jmsy.2022.03.015"},{"doi-asserted-by":"publisher","key":"ref105","DOI":"10.1109\/SMDS53860.2021.00021"},{"doi-asserted-by":"publisher","key":"ref106","DOI":"10.1109\/RAMS51492.2024.10457600"},{"doi-asserted-by":"publisher","key":"ref107","DOI":"10.1038\/s41746-022-00611-y"},{"key":"ref108","first-page":"1","article-title":"Domain shifts in reinforcement learning: Identifying disturbances in environments","volume-title":"Proc. AISafety@ IJCAI","author":"Haider","year":"2021"},{"doi-asserted-by":"publisher","key":"ref109","DOI":"10.1145\/3580305.3599802"},{"key":"ref110","first-page":"1","article-title":"Monitoring machine learning-based risk prediction algorithms in the presence of performativity","volume-title":"Int. Conf. Artif. Intell. Statist.","author":"Feng","year":"2024"},{"doi-asserted-by":"publisher","key":"ref111","DOI":"10.1007\/978-3-031-43898-1_38"},{"doi-asserted-by":"publisher","key":"ref112","DOI":"10.1007\/978-3-030-44584-3_36"},{"doi-asserted-by":"publisher","key":"ref113","DOI":"10.1145\/3420034"},{"doi-asserted-by":"publisher","key":"ref114","DOI":"10.2202\/1557-4679.1203"},{"doi-asserted-by":"publisher","key":"ref115","DOI":"10.1007\/978-3-031-46002-9_15"},{"doi-asserted-by":"publisher","key":"ref116","DOI":"10.1007\/978-3-030-93736-2_46"},{"doi-asserted-by":"publisher","key":"ref117","DOI":"10.1145\/1833280.1833283"},{"doi-asserted-by":"publisher","key":"ref118","DOI":"10.1007\/978-3-031-34344-5_37"},{"doi-asserted-by":"publisher","key":"ref119","DOI":"10.1007\/978-3-031-40953-0_32"},{"doi-asserted-by":"publisher","key":"ref120","DOI":"10.1109\/WACV57701.2024.00212"},{"doi-asserted-by":"publisher","key":"ref121","DOI":"10.1007\/s10270-023-01090-9"},{"doi-asserted-by":"publisher","key":"ref122","DOI":"10.1145\/3580305.3599454"},{"doi-asserted-by":"publisher","key":"ref123","DOI":"10.1007\/978-3-031-19992-9_1"},{"doi-asserted-by":"publisher","key":"ref124","DOI":"10.1145\/3340531.3412705"},{"doi-asserted-by":"publisher","key":"ref125","DOI":"10.1145\/3468264.3473117"},{"doi-asserted-by":"publisher","key":"ref126","DOI":"10.1145\/3340482.3342743"},{"doi-asserted-by":"publisher","key":"ref127","DOI":"10.1145\/3394486.3403205"},{"key":"ref128","first-page":"234","article-title":"Novel contract-based runtime explainability framework for end-to-end ensemble machine learning serving","volume-title":"Proc. IEEE\/ACM 3rd Int. Conf. AI Eng.-Softw. Eng. AI","author":"Nguyen","year":"2024"},{"key":"ref129","first-page":"1","article-title":"Jenga - A framework to study the impact of data errors on the predictions of machine learning models","volume-title":"Proc. Adv. Database Technol. EDBT","author":"Schelter","year":"2021"},{"doi-asserted-by":"publisher","key":"ref130","DOI":"10.1145\/3318464.3380604"},{"doi-asserted-by":"publisher","key":"ref131","DOI":"10.1109\/OJCOMS.2023.3337850"},{"doi-asserted-by":"publisher","key":"ref132","DOI":"10.1109\/ACCESS.2021.3125920"},{"doi-asserted-by":"publisher","key":"ref133","DOI":"10.1109\/JETCAS.2021.3076101"},{"doi-asserted-by":"publisher","key":"ref134","DOI":"10.1109\/COMPSAC54236.2022.00168"},{"key":"ref135","first-page":"3962","article-title":"MLDemon: Deployment monitoring for machine learning systems","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Ginart","year":"2022"},{"doi-asserted-by":"publisher","key":"ref136","DOI":"10.1145\/3514221.3517904"},{"key":"ref137","first-page":"178","article-title":"Ml-on-rails: Safeguarding machine learning models in software systems\u2014A case study","volume-title":"Proc. IEEE\/ACM 3rd Int. Conf. AI Eng.-Softw. Eng. AI","author":"Abdelkader","year":"2024"},{"doi-asserted-by":"publisher","key":"ref138","DOI":"10.1145\/3611093"},{"doi-asserted-by":"publisher","key":"ref139","DOI":"10.1007\/s11634-023-00568-0"},{"key":"ref140","first-page":"1","article-title":"Taming false positives in out-of-distribution detection with human feedback","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Vishwakarma","year":"2024"},{"doi-asserted-by":"publisher","key":"ref141","DOI":"10.1016\/j.jacr.2021.08.022"},{"doi-asserted-by":"publisher","key":"ref142","DOI":"10.3233\/FAIA220202"},{"doi-asserted-by":"publisher","key":"ref143","DOI":"10.1145\/3479569"},{"doi-asserted-by":"publisher","key":"ref144","DOI":"10.1109\/ESEM.2019.8870157"},{"doi-asserted-by":"publisher","key":"ref145","DOI":"10.1145\/3613905.3651120"},{"doi-asserted-by":"publisher","key":"ref146","DOI":"10.5220\/0011660300003393"},{"key":"ref147","doi-asserted-by":"crossref","DOI":"10.31219\/osf.io\/wm2av","article-title":"Accelerating systematic reviews with large language models: Current practices and recommendations","author":"Hua","year":"2024"},{"doi-asserted-by":"publisher","key":"ref148","DOI":"10.1007\/978-3-319-17461-7_100"},{"year":"2025","article-title":"Tensorflow extended (TFX)","key":"ref149"},{"year":"2025","article-title":"Mlflow: An open source platform for the machine learning lifecycle","key":"ref150"},{"year":"2025","article-title":"Apache airflow","key":"ref151"},{"year":"2025","article-title":"Amundsen - Data discovery and metadata platform","key":"ref152"},{"year":"2025","article-title":"Datahub - Metadata platform for the modern data stack","key":"ref153"},{"doi-asserted-by":"publisher","key":"ref154","DOI":"10.1145\/3674805.3686691"},{"doi-asserted-by":"publisher","key":"ref155","DOI":"10.1016\/j.infsof.2018.10.006"},{"first-page":"1","article-title":"Embedding leaderboard","key":"ref156"},{"doi-asserted-by":"publisher","key":"ref157","DOI":"10.1016\/j.ejor.2011.05.007"},{"doi-asserted-by":"publisher","key":"ref158","DOI":"10.1145\/3468264.3468536"}],"container-title":["IEEE Transactions on Software Engineering"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/32\/11298241\/11142468.pdf?arnumber=11142468","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,18]],"date-time":"2025-12-18T10:23:18Z","timestamp":1766053398000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11142468\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,12]]},"references-count":158,"journal-issue":{"issue":"12"},"URL":"https:\/\/doi.org\/10.1109\/tse.2025.3602520","relation":{},"ISSN":["0098-5589","1939-3520","2326-3881"],"issn-type":[{"type":"print","value":"0098-5589"},{"type":"electronic","value":"1939-3520"},{"type":"electronic","value":"2326-3881"}],"subject":[],"published":{"date-parts":[[2025,12]]}}}