{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,7]],"date-time":"2026-10-07T01:24:34Z","timestamp":1791336274954,"version":"4.3.1"},"reference-count":158,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"12","license":[{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IIEEE Trans. Software Eng."],"published-print":{"date-parts":[[2025,12]]},"DOI":"10.1109\/tse.2025.3602520","type":"journal-article","created":{"date-parts":[[2025,8,26]],"date-time":"2025-08-26T19:21:36Z","timestamp":1756236096000},"page":"3218-3246","source":"Crossref","is-referenced-by-count":6,"title":["From Tea Leaves to System Maps: A Survey and Framework on Context-Aware Machine Learning Monitoring"],"prefix":"10.1109","volume":"51","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-1335-6559","authenticated-orcid":false,"given":"Joran","family":"Leest","sequence":"first","affiliation":[{"name":"Department of Computer Science, Faculty of Sciences, Vrije Universiteit Amsterdam, Amsterdam, HV, The Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7194-3159","authenticated-orcid":false,"given":"Claudia","family":"Raibulet","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Faculty of Sciences, Vrije Universiteit Amsterdam, Amsterdam, HV, The Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2234-0845","authenticated-orcid":false,"given":"Patricia","family":"Lago","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Faculty of Sciences, Vrije Universiteit Amsterdam, Amsterdam, HV, The Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9333-7101","authenticated-orcid":false,"given":"Ilias","family":"Gerostathopoulos","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Faculty of Sciences, Vrije Universiteit Amsterdam, Amsterdam, HV, The Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-022-15245-z"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1145\/3653697"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/3240323.3240370"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2017.8258038"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1145\/2523813"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2018.2876857"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1002\/widm.1327"},{"key":"ref8","article-title":"Leveraging unlabeled data to predict out-of-distribution performance","author":"Garg","year":"2022"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00117"},{"key":"ref10","first-page":"19274","article-title":"Agreement-on-the-line: Predicting the performance of neural networks under distribution shift","volume":"35","author":"Baek","year":"2022","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref11","first-page":"1","article-title":"Data validation for machine learning","volume-title":"Proc. MLSys","author":"Breck","year":"2019"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.14778\/3229863.3229867"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1145\/3583780.3614786"},{"key":"ref14","first-page":"61","article-title":"Automating data quality validation for dynamic data ingestion","author":"Redyuk","year":"2021"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v10i1.21997"},{"key":"ref16","first-page":"1","article-title":"Hidden technical debt in machine learning systems","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"28","author":"Sculley","year":"2015"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330744"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/3533378"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP52600.2021.00034"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1145\/3510457.3513058"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2022.3161161"},{"key":"ref23","volume-title":"Designing Machine Learning Systems","author":"Huyen","year":"2022"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/3318464.3384707"},{"key":"ref25","first-page":"1","article-title":"Overton: A data system for monitoring and improving machine-learned products","volume-title":"Proc. 10th Annu. Conf. Innovative Data Syst. Res. (CIDR)","author":"R\u00e9","year":"2020"},{"key":"ref26","first-page":"1","article-title":"Why did the distribution change?","volume-title":"Proc. Mach. Learn. Res.","author":"Kailash","year":"2021"},{"key":"ref27","first-page":"1","article-title":"Diagnosing failures of fairness transfer across distribution shift in real-world medical settings","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"35","author":"Schrouff","year":"2022"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1111\/exsy.12059"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1145\/3639476.3639771"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1016\/j.dsm.2022.07.004"},{"key":"ref31","article-title":"Inferring concept drift without labeled data","year":"2021"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-015-0448-4"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/3522664.3528590"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.14778\/3565838.3565853"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1145\/3514094.3534157"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1111\/coin.12520"},{"key":"ref37","first-page":"1","article-title":"Labelless concept drift detection and explanation","volume-title":"Proc. Workshop Robust AI Financial Services, Data, Fairness, Explainability, Trustworthiness, Privacy (NeurIPS)","author":"Zheng","year":"2019"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-27615-7_17"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2021.3057686"},{"key":"ref40","doi-asserted-by":"crossref","first-page":"9845","DOI":"10.1007\/978-981-16-8193-6","article-title":"Sequential covariate shift detection using classifier two-sample tests","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Jang","year":"2022"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/VIS49827.2021.9623289"},{"key":"ref42","article-title":"Monitoring bias and fairness in machine learning models: A review","author":"Bacelar","year":"2021","journal-title":"ScienceOpen Preprints"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445865"},{"key":"ref44","first-page":"1617","article-title":"Mandoline: Model evaluation under distribution shift","volume-title":"Int. Conf. Mach. Learn.","author":"Chen","year":"2021"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-26250-1_29"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-05434-1_1"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2021.111096"},{"key":"ref48","first-page":"160","article-title":"Runaway feedback loops in predictive policing","volume-title":"Proc. Conf. Fairness, Accountability Transparency","author":"Ensign","year":"2018"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.3991\/ijim.v17i23.43479"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/OCIT59427.2023.10431343"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2023.3262138"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2962027"},{"key":"ref53","article-title":"Test & evaluation best practices for machine learning-enabled systems","author":"Chandrasekaran","year":"2023"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/SaTML54575.2023.00038"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1016\/j.patter.2023.100780"},{"key":"ref56","first-page":"1","article-title":"Context: The missing piece in the machine learning lifecycle","volume-title":"Proc. Common Model Infrastructure Workshop","author":"Garcia","year":"2018"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-020-0262-2"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1207\/s15327051hci16234_02"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1504\/IJAHUC.2007.014070"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.5220\/0010818200003119"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/ICSA.2017.26"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2019.04.011"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1016\/j.datak.2017.03.008"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-24465-5_7"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.3233\/IDA-194515"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2013.419"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1145\/3661826"},{"key":"ref68","first-page":"587","article-title":"Designing monitoring strategies for deployed machine learning algorithms: Navigating performativity through a causal lens","volume-title":"Proc. Causal Learn. Reasoning","author":"Feng","year":"2024"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1109\/CDC49753.2023.10383566"},{"key":"ref70","first-page":"4087","article-title":"Context-aware drift detection","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Cobb","year":"2022"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1145\/3526073.3527590"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1109\/CAIN58948.2023.00009"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1145\/3531232.3531261"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.32473\/flairs.36.133240"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1145\/3578356.3592593"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-20485-3_17"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2013.2271915"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1109\/DeepTest52559.2021.00007"},{"key":"ref79","first-page":"1","article-title":"Why did the model fail?\u2019: Attributing model performance changes to distribution shifts","volume-title":"Proc. Mach. Learn. Res.","author":"Zhang","year":"2023"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1145\/3076246.3076251"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657834"},{"key":"ref82","article-title":"The fastest and most powerful platform for building AI products","year":"2025"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.137"},{"issue":"2","key":"ref84","first-page":"1","article-title":"Miss intensified customer churn prediction: Connectivity with weighted multi-layer perceptron and enhanced multipath back propagation: English","volume":"4","author":"Panimalar","year":"2025","journal-title":"Int. J. Adv. Sci. Computer Appl."},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1007\/s42979-024-02722-7"},{"key":"ref86","article-title":"End-to-end blueprint for customer churn modeling and prediction \u2013 Part 1","author":"Blog","year":"2025"},{"key":"ref87","article-title":"Data pipeline architecture for churn prediction: Automating data flow with aws services","author":"Analytics","year":"2025"},{"key":"ref88","article-title":"Churn prediction for game developers using google analytics 4 (ga4) and bigquery ML","author":"Developers","year":"2025"},{"key":"ref89","article-title":"Subscriber churn prediction and retention on aws","author":"Services","year":"2025"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2016.0087"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1145\/3617694.3623239"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1145\/3399579.3399868"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-29786-1_16"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1145\/3593013.3594028"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1145\/3366423.3380306"},{"key":"ref96","first-page":"1","article-title":"On the (in)feasibility of ml backdoor detection as an hypothesis testing problem","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Pichler","year":"2024"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE48307.2020.00140"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-44267-4_15"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1016\/j.clsr.2024.105986"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.37934\/araset.40.2.127139"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-022-00845-0"},{"key":"ref102","first-page":"1","article-title":"Estimating and explaining model performance when both covariates and labels shift","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"35","author":"Chen","year":"2022"},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.1109\/ICTAI56018.2022.00009"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.1016\/j.jmsy.2022.03.015"},{"key":"ref105","doi-asserted-by":"publisher","DOI":"10.1109\/SMDS53860.2021.00021"},{"key":"ref106","doi-asserted-by":"publisher","DOI":"10.1109\/RAMS51492.2024.10457600"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-022-00611-y"},{"key":"ref108","first-page":"1","article-title":"Domain shifts in reinforcement learning: Identifying disturbances in environments","volume-title":"Proc. AISafety@ IJCAI","author":"Haider","year":"2021"},{"key":"ref109","doi-asserted-by":"publisher","DOI":"10.1145\/3580305.3599802"},{"key":"ref110","first-page":"1","article-title":"Monitoring machine learning-based risk prediction algorithms in the presence of performativity","volume-title":"Int. Conf. Artif. Intell. Statist.","author":"Feng","year":"2024"},{"key":"ref111","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-43898-1_38"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-44584-3_36"},{"key":"ref113","doi-asserted-by":"publisher","DOI":"10.1145\/3420034"},{"key":"ref114","doi-asserted-by":"publisher","DOI":"10.2202\/1557-4679.1203"},{"key":"ref115","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-46002-9_15"},{"key":"ref116","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-93736-2_46"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.1145\/1833280.1833283"},{"key":"ref118","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-34344-5_37"},{"key":"ref119","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-40953-0_32"},{"key":"ref120","doi-asserted-by":"publisher","DOI":"10.1109\/WACV57701.2024.00212"},{"key":"ref121","doi-asserted-by":"publisher","DOI":"10.1007\/s10270-023-01090-9"},{"key":"ref122","doi-asserted-by":"publisher","DOI":"10.1145\/3580305.3599454"},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19992-9_1"},{"key":"ref124","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3412705"},{"key":"ref125","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3473117"},{"key":"ref126","doi-asserted-by":"publisher","DOI":"10.1145\/3340482.3342743"},{"key":"ref127","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3403205"},{"key":"ref128","first-page":"234","article-title":"Novel contract-based runtime explainability framework for end-to-end ensemble machine learning serving","volume-title":"Proc. IEEE\/ACM 3rd Int. Conf. AI Eng.-Softw. Eng. AI","author":"Nguyen","year":"2024"},{"key":"ref129","first-page":"1","article-title":"Jenga - A framework to study the impact of data errors on the predictions of machine learning models","volume-title":"Proc. Adv. Database Technol. EDBT","author":"Schelter","year":"2021"},{"key":"ref130","doi-asserted-by":"publisher","DOI":"10.1145\/3318464.3380604"},{"key":"ref131","doi-asserted-by":"publisher","DOI":"10.1109\/OJCOMS.2023.3337850"},{"key":"ref132","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2021.3125920"},{"key":"ref133","doi-asserted-by":"publisher","DOI":"10.1109\/JETCAS.2021.3076101"},{"key":"ref134","doi-asserted-by":"publisher","DOI":"10.1109\/COMPSAC54236.2022.00168"},{"key":"ref135","first-page":"3962","article-title":"MLDemon: Deployment monitoring for machine learning systems","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Ginart","year":"2022"},{"key":"ref136","doi-asserted-by":"publisher","DOI":"10.1145\/3514221.3517904"},{"key":"ref137","first-page":"178","article-title":"Ml-on-rails: Safeguarding machine learning models in software systems\u2014A case study","volume-title":"Proc. IEEE\/ACM 3rd Int. Conf. AI Eng.-Softw. Eng. AI","author":"Abdelkader","year":"2024"},{"key":"ref138","doi-asserted-by":"publisher","DOI":"10.1145\/3611093"},{"key":"ref139","doi-asserted-by":"publisher","DOI":"10.1007\/s11634-023-00568-0"},{"key":"ref140","first-page":"1","article-title":"Taming false positives in out-of-distribution detection with human feedback","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Vishwakarma","year":"2024"},{"key":"ref141","doi-asserted-by":"publisher","DOI":"10.1016\/j.jacr.2021.08.022"},{"key":"ref142","doi-asserted-by":"publisher","DOI":"10.3233\/FAIA220202"},{"key":"ref143","doi-asserted-by":"publisher","DOI":"10.1145\/3479569"},{"key":"ref144","doi-asserted-by":"publisher","DOI":"10.1109\/ESEM.2019.8870157"},{"key":"ref145","doi-asserted-by":"publisher","DOI":"10.1145\/3613905.3651120"},{"key":"ref146","doi-asserted-by":"publisher","DOI":"10.5220\/0011660300003393"},{"key":"ref147","doi-asserted-by":"crossref","DOI":"10.31219\/osf.io\/wm2av","article-title":"Accelerating systematic reviews with large language models: Current practices and recommendations","author":"Hua","year":"2024"},{"key":"ref148","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-17461-7_100"},{"key":"ref149","article-title":"Tensorflow extended (TFX)","year":"2025"},{"key":"ref150","article-title":"Mlflow: An open source platform for the machine learning lifecycle","year":"2025"},{"key":"ref151","article-title":"Apache airflow","year":"2025"},{"key":"ref152","article-title":"Amundsen - Data discovery and metadata platform","year":"2025"},{"key":"ref153","article-title":"Datahub - Metadata platform for the modern data stack","year":"2025"},{"key":"ref154","doi-asserted-by":"publisher","DOI":"10.1145\/3674805.3686691"},{"key":"ref155","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2018.10.006"},{"key":"ref156","first-page":"1","article-title":"Embedding leaderboard"},{"key":"ref157","doi-asserted-by":"publisher","DOI":"10.1016\/j.ejor.2011.05.007"},{"key":"ref158","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468536"}],"container-title":["IEEE Transactions on Software Engineering"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/32\/11298241\/11142468.pdf?arnumber=11142468","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,18]],"date-time":"2025-12-18T10:23:18Z","timestamp":1766053398000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11142468\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,12]]},"references-count":158,"journal-issue":{"issue":"12"},"URL":"https:\/\/doi.org\/10.1109\/tse.2025.3602520","relation":{},"ISSN":["0098-5589","1939-3520","2326-3881"],"issn-type":[{"value":"0098-5589","type":"print"},{"value":"1939-3520","type":"electronic"},{"value":"2326-3881","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,12]]}}}