{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,17]],"date-time":"2026-07-17T15:26:56Z","timestamp":1784302016873,"version":"3.55.0"},"reference-count":61,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2022,1,31]],"date-time":"2022-01-31T00:00:00Z","timestamp":1643587200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,1,31]],"date-time":"2022-01-31T00:00:00Z","timestamp":1643587200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["The VLDB Journal"],"published-print":{"date-parts":[[2022,9]]},"DOI":"10.1007\/s00778-021-00726-w","type":"journal-article","created":{"date-parts":[[2022,1,31]],"date-time":"2022-01-31T02:02:17Z","timestamp":1643594537000},"page":"1103-1126","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":24,"title":["Data distribution debugging in machine learning pipelines"],"prefix":"10.1007","volume":"31","author":[{"given":"Stefan","family":"Grafberger","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Paul","family":"Groth","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Julia","family":"Stoyanovich","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sebastian","family":"Schelter","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2022,1,31]]},"reference":[{"key":"726_CR1","unstructured":"Agarwal, A., Beygelzimer, A., Dudik, M., Langford, J., Wallach, H.: A reductions approach to fair classification. In: FAT* (2017)"},{"key":"726_CR2","doi-asserted-by":"crossref","unstructured":"Albarghouthi, A., Vinitsky, S: Fairness-aware programming. In: FAT* (2019)","DOI":"10.1145\/3287560.3287588"},{"key":"726_CR3","doi-asserted-by":"crossref","unstructured":"Amsterdamer, Y., Davidson, S.B., Deutch, D., Milo, T, Stoyanovich, J. Tannen, V. Enabling database-style workflow provenance. In: PVLDB, Putting Lipstick on Pig (2011)","DOI":"10.14778\/2095686.2095693"},{"key":"726_CR4","doi-asserted-by":"crossref","unstructured":"Amsterdamer, Y., Deutch, D., Tannen, V: Provenance for aggregate queries. In: PODS (2011)","DOI":"10.1145\/1989284.1989302"},{"key":"726_CR5","doi-asserted-by":"crossref","unstructured":"Angelino, E., Yamins, D., Seltzer, M.: Starflow: a script-centric data analysis environment. In: Provenance and Annotation of Data and Processes (2010)","DOI":"10.1007\/978-3-642-17819-1_27"},{"key":"726_CR6","unstructured":"Angwin, J., Larson, J., Mattu, S., Kirchner, L.: Machine bias. (propublica) (2016)"},{"key":"726_CR7","doi-asserted-by":"crossref","unstructured":"Asudeh, A., Jin, Z., Jagadish, H.V.: Assessing and remedying coverage for a given dataset. In: ICDE (2019)","DOI":"10.1109\/ICDE.2019.00056"},{"key":"726_CR8","unstructured":"Bellamy, R.K.E., Dey, K., Hind, M., Hoffman, S.C., Houde, S., et al.: AI fairness 360: an extensible toolkit for detecting, understanding, and mitigating unwanted algorithmic bias (2018)"},{"key":"726_CR9","doi-asserted-by":"crossref","unstructured":"Brachmann, M., Bautista, C., Castelo, S., Feng, S., Freire, J., et al.: Data debugging and exploration with vizier. In: SIGMOD, Su Feng (2019)","DOI":"10.1145\/3299869.3320246"},{"key":"726_CR10","unstructured":"Breck, E., Zinkevich, M., Whang, S., Roy, S.: Data validation for machine learning. In: SysML, Neoklis Polyzotis (2019)"},{"key":"726_CR11","doi-asserted-by":"crossref","unstructured":"Brun, Y., Meliou, A.: Software fairness. In: ESEC\/FSE (2018)","DOI":"10.1145\/3236024.3264838"},{"key":"726_CR12","unstructured":"Chen, I., Johansson, F.D., Sontag, D.: Why is my classifier discriminatory? In: NeurIPS (2018)"},{"key":"726_CR13","doi-asserted-by":"crossref","unstructured":"Cheney, J., Chiticariu, L., Tan, W.C: Provenance in Databases: Why, How, and Where. Found. Trends Databases, vol. 1, no. 4 (2009)","DOI":"10.1561\/1900000006"},{"key":"726_CR14","doi-asserted-by":"crossref","unstructured":"Chouldechova, A., Roth, A.: A snapshot of the frontiers of fairness in machine learning. In: CACM, vol 63, no. 5 (2020)","DOI":"10.1145\/3376898"},{"key":"726_CR15","doi-asserted-by":"crossref","unstructured":"Galhotra, S., Brun, Y., Meliou, A: Testing software for discrimination. In: ESEC\/FSE, Fairness Testing (2017)","DOI":"10.1145\/3106237.3106277"},{"key":"726_CR16","unstructured":"Gebru, T., Morgenstern, J., Vecchione, B. et\u00a0al.: Datasheets for datasets (2018)"},{"key":"726_CR17","unstructured":"Grafberger, S., Stoyanovich, J., Schelter, S.: Lightweight inspection of data preprocessing in native machine learning pipelines. In: Conference on Innovative Data Systems Research (CIDR) (2021)"},{"key":"726_CR18","doi-asserted-by":"crossref","unstructured":"Green, T.J., Karvounarakis, G., Tannen, V.: Provenance semirings. In: PODS (2007)","DOI":"10.1145\/1265530.1265535"},{"key":"726_CR19","doi-asserted-by":"crossref","unstructured":"Herschel, M., Diestelk\u00e4mper, R., Lahmar, H.B.: A survey on provenance: What for? What form? What from? VLDBJ 26(6) (2017)","DOI":"10.1007\/s00778-017-0486-1"},{"key":"726_CR20","doi-asserted-by":"crossref","unstructured":"Hutton, G.: A tutorial on the universality and expressiveness of fold. J. Funct. Program, 8 (1999)","DOI":"10.1017\/S0956796899003500"},{"key":"726_CR21","unstructured":"Hynes, N., Sculley, D., Terry, M. The data linter: lightweight, automated sanity checking for ml data sets. In: MLSystems workshop at NeurIPS (2017)"},{"key":"726_CR22","doi-asserted-by":"crossref","unstructured":"Interlandi, M., Shah, K., et\u00a0al. Titian: data provenance support in spark. In: VLDB (2015)","DOI":"10.14778\/2850583.2850595"},{"key":"726_CR23","unstructured":"Jindal, A., Emani, K.V., Daum, M., Poppe, O., et\u00a0al: Magpie: python at speed and scale using cloud backends. In: CIDR (2021)"},{"key":"726_CR24","doi-asserted-by":"crossref","unstructured":"Logothetis, D., De, S., Yocum, K: Scalable lineage capture for debugging disc analytics. In: SoCC (2013)","DOI":"10.1145\/2523616.2523619"},{"key":"726_CR25","doi-asserted-by":"crossref","unstructured":"Louren\u00e7o, R., Freire, J., Shasha, D.: A system for debugging computational pipelines. In: SIGMOD, Bugdoc (2020)","DOI":"10.1145\/3318464.3384692"},{"key":"726_CR26","unstructured":"Madden, S., Ouzzani, M., Tang, N., Stonebraker, M.: Dagger: a data (not code) debugger. In: CIDR (2020)"},{"key":"726_CR27","unstructured":"McPhillips, T.M., Song, T., Kolisnik, T., et al.: Yesworkflow: a user-oriented, language-independent tool for recovering workflow information from scripts. In: CoRR, abs\/1502.02403 (2015)"},{"issue":"1","key":"726_CR28","first-page":"1235","volume":"17","author":"X Meng","year":"2016","unstructured":"Meng, X., Bradley, J., Yavuz, B., Sparks, E., Venkataraman, S., Liu, D., Freeman, J., Tsai, D.B., Amde, M., Owen, S., et al.: Mllib: machine learning in apache spark. JMLR 17(1), 1235\u20131241 (2016)","journal-title":"JMLR"},{"key":"726_CR29","doi-asserted-by":"crossref","unstructured":"Miao, H., Li, A., Davis, L.S., Deshpande, A.: Towards unified data and lifecycle management for deep learning. In: ICDE, pp. 571\u2013582 (2017)","DOI":"10.1109\/ICDE.2017.112"},{"key":"726_CR30","doi-asserted-by":"crossref","unstructured":"Miao, H., Deshpande, A.: Provdb: provenance-enabled lifecycle management of collaborative data analysis workflows. IEEE Data Eng. Bull 41 (2018)","DOI":"10.1145\/3077257.3077267"},{"key":"726_CR31","doi-asserted-by":"crossref","unstructured":"Moreau, L.: The foundations for provenance on the web. Found. Trends Web Sci. 2(2\u20133), 29, 99\u2013241 (2010)","DOI":"10.1561\/1800000010"},{"key":"726_CR32","doi-asserted-by":"crossref","unstructured":"Mitchell, M., et al.: Model cards for model reporting. In: FAT* (2019)","DOI":"10.1145\/3287560.3287596"},{"key":"726_CR33","unstructured":"Murta, L., Braganholo, V., Chirigati, F., Koop, D., Freire, J.: noworkflow: capturing and analyzing provenance of scripts. In: VLDB (2017)"},{"key":"726_CR34","unstructured":"Namaki, M.H., Floratou, A., Psallidas, F., Krishnan, S., Agrawal, A., Wu, Y.: Tracking provenance in data science scripts. In: KDD, Vamsa (2020)"},{"key":"726_CR35","doi-asserted-by":"crossref","unstructured":"Olston, C., Reed, B.: Inspector gadget: A framework for custom monitoring and debugging of distributed dataflows. In: SIGMOD (2011)","DOI":"10.1145\/1989323.1989459"},{"key":"726_CR36","unstructured":"Ormenisan, A.A., Meister, M., Buso, F., Andersson, R., Haridi, S., Dowling, J.: Time travel and provenance for machine learning pipelines. In: OpML at USENIX (2020)"},{"key":"726_CR37","unstructured":"Pedregosa, F., Varoquaux, G., Gramfort, A. et\u00a0al.: Scikit-learn: Machine learning in python. In: JMLR, vol. 12 (2011)"},{"key":"726_CR38","doi-asserted-by":"crossref","unstructured":"Pei, K., Cao, Y., Yang, J., Jana, S.: Deepxplore. In: SOSP (2017)","DOI":"10.1145\/3132747.3132785"},{"key":"726_CR39","doi-asserted-by":"crossref","unstructured":"Petersohn, D., Macke, S., Xin, D., Ma, W., Lee, D., Mo, X., Gonzalez, J.E., Hellerstein, J.M., Joseph, A.D., Parameswaran, A: Towards scalable dataframe systems. In: VLDB (2020)","DOI":"10.14778\/3407790.3407807"},{"key":"726_CR40","doi-asserted-by":"crossref","unstructured":"Pimentel, J.F., Murta, L., Braganholo, V. and Freire, J.: noworkflow: a tool for collecting, analyzing, and managing provenance from python scripts. In: PVLDB (2017)","DOI":"10.14778\/3137765.3137789"},{"key":"726_CR41","doi-asserted-by":"crossref","unstructured":"Polyzotis, N., Roy, S., Whang, S.E., Zinkevich, M.: Data lifecycle challenges in production machine learning: a survey. In: SIGMOD Record (2018)","DOI":"10.1145\/3035918.3054782"},{"key":"726_CR42","unstructured":"Polyzotis, N., Whang, S., Kraska, T.K. and Chung, Y.: Automated data slicing for model validation. In: ICDE, Slice finder (2019)"},{"key":"726_CR43","doi-asserted-by":"crossref","unstructured":"Psallidas, F., Wu, E.: Smoke: Fine-grained lineage at interactive speed. In: VLDB (2018)","DOI":"10.14778\/3184470.3184475"},{"key":"726_CR44","unstructured":"Psallidas, F., Zhu, Y., Karlas, B., et\u00a0al: Data science through the looking glass and what we found there (2019)"},{"key":"726_CR45","unstructured":"Raasveldt, M., M\u00fchleisen, H.: Data management for data science-towards embedded analytics. In: CIDR (2020)"},{"key":"726_CR46","unstructured":"Schelter, S., Boese, J.H., Kirschnick, J., Klein, T., Seufert, S.: Automatically tracking metadata and provenance of machine learning experiments. In: ML Systems Workshop at NeurIPS (2017)"},{"key":"726_CR47","unstructured":"Schelter, S., He, Y., Khilnani, J. and Stoyanovich, J.: Fairprep: Promoting data to a first-class citizen in studies on fairness-enhancing interventions. In: EDBT (2019)"},{"key":"726_CR48","doi-asserted-by":"crossref","unstructured":"Schelter, S., Lange, D., Schmidt, P., Celikel, M., Biessmann, F., Grafberger, A: Automating large-scale data quality verification. In: PVLDB, Meltem Celikel (2018)","DOI":"10.14778\/3229863.3229867"},{"key":"726_CR49","first-page":"39","volume":"43","author":"S Sebastian","year":"2020","unstructured":"Sebastian, S.: Stoyanovich, J: Taming technical bias in machine learning pipelines. IEEE Data Eng. Bull. 43, 39\u201350 (2020)","journal-title":"IEEE Data Eng. Bull."},{"key":"726_CR50","doi-asserted-by":"crossref","unstructured":"Sparks, E.R., Venkataraman, S., Kaftan, T., Franklin, M.J., Recht, B.: Keystoneml: Optimizing pipelines for large-scale advanced analytics. In: ICDE (2017)","DOI":"10.1109\/ICDE.2017.109"},{"issue":"3","key":"726_CR51","first-page":"13","volume":"42","author":"J Stoyanovich","year":"2019","unstructured":"Stoyanovich, J., Howe, B.: Nutritional labels for data and models. IEEE Data Eng. Bull. 42(3), 13\u201323 (2019)","journal-title":"IEEE Data Eng. Bull."},{"key":"726_CR52","doi-asserted-by":"crossref","unstructured":"Stoyanovich, J., Howe, B., Jagadish, H.V.: Responsible data management. In: VLDB (2020)","DOI":"10.14778\/3415478.3415570"},{"issue":"4","key":"726_CR53","first-page":"16","volume":"41","author":"M Vartak","year":"2018","unstructured":"Vartak, M., Madden, S.: Modeldb: opportunities and challenges in managing machine learning models. IEEE Data Eng. Bull. 41(4), 16\u201325 (2018)","journal-title":"IEEE Data Eng. Bull."},{"key":"726_CR54","unstructured":"Vartak, M., Joana, Trindade, J.M., Madden, S., Zaharia, M: A system to store and query model intermediates for model diagnosis. In: SIGMOD (2018)"},{"key":"726_CR55","unstructured":"Wikipedia. Monkey patch. https:\/\/en.wikipedia.org\/wiki\/Monkey_patch (2021). Accessed 9 Sept 2021"},{"issue":"1","key":"726_CR56","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/sdata.2016.18","volume":"3","author":"MD Wilkinson","year":"2016","unstructured":"Wilkinson, M.D., Dumontier, M., Aalbersberg, I.J.J., Appleton, G., Axton, M., Baak, A., Blomberg, N., et al.: The fair guiding principles for scientific data management and stewardship. Sci. Data 3(1), 1\u20139 (2016)","journal-title":"Sci. Data"},{"key":"726_CR57","unstructured":"Yan, Z., Tannen, V., Ives, Z.G.: Fine-grained provenance for linear algebra operators. In: TaPP (2016)"},{"key":"726_CR58","unstructured":"Yang, K., Huang, B., Stoyanovich, J., Schelter, S.: Fairness-aware instrumentation of preprocessing pipelines for machine learning. In: HILDA Workshop at SIGMOD (2020)"},{"key":"726_CR59","doi-asserted-by":"crossref","unstructured":"Yang, K., Stoyanovich, J., Asudeh, A., Howe, B., Jagadish, H.V., Miklau, G.: A nutritional label for rankings. In: SIGMOD (2018)","DOI":"10.1145\/3183713.3193568"},{"issue":"4","key":"726_CR60","first-page":"39","volume":"41","author":"M Zaharia","year":"2018","unstructured":"Zaharia, M., Chen, A., Davidson, A., Ghodsi, A., Hong, S.A., Konwinski, A., Murching, S., et al.: Accelerating the machine learning lifecycle with MLflow. IEEE Data Eng. Bull. 41(4), 39\u201345 (2018)","journal-title":"IEEE Data Eng. Bull."},{"key":"726_CR61","doi-asserted-by":"crossref","unstructured":"Zhang, Z., Sparks, E.R., Franklin, M.J.: Diagnosing machine learning pipelines with fine-grained lineage. In: HPDC (2017)","DOI":"10.1145\/3078597.3078603"}],"container-title":["The VLDB Journal"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00778-021-00726-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00778-021-00726-w\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00778-021-00726-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,8,28]],"date-time":"2022-08-28T06:10:48Z","timestamp":1661667048000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00778-021-00726-w"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,1,31]]},"references-count":61,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2022,9]]}},"alternative-id":["726"],"URL":"https:\/\/doi.org\/10.1007\/s00778-021-00726-w","relation":{},"ISSN":["1066-8888","0949-877X"],"issn-type":[{"value":"1066-8888","type":"print"},{"value":"0949-877X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,1,31]]},"assertion":[{"value":"27 February 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"9 September 2021","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"3 December 2021","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"31 January 2022","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}