{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,30]],"date-time":"2026-03-30T22:30:29Z","timestamp":1774909829728,"version":"3.50.1"},"publisher-location":"Cham","reference-count":36,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031236327","type":"print"},{"value":"9783031236334","type":"electronic"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-23633-4_13","type":"book-chapter","created":{"date-parts":[[2023,1,30]],"date-time":"2023-01-30T07:05:49Z","timestamp":1675062349000},"page":"170-185","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Evaluation of\u00a0the\u00a0Limit of\u00a0Detection in\u00a0Network Dataset Quality Assessment with\u00a0PerQoDA"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8087-790X","authenticated-orcid":false,"given":"Katarzyna","family":"Wasielewska","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4737-8735","authenticated-orcid":false,"given":"Dominik","family":"Soukup","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7794-9511","authenticated-orcid":false,"given":"Tom\u00e1\u0161","family":"\u010cejka","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9804-8122","authenticated-orcid":false,"given":"Jos\u00e9","family":"Camacho","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,1,31]]},"reference":[{"key":"13_CR1","doi-asserted-by":"publisher","unstructured":"Anderson, M.J.: Permutational multivariate analysis of variance (PERMANOVA), pp. 1\u201315. Wiley (2017). https:\/\/doi.org\/10.1002\/9781118445112.stat07841","DOI":"10.1002\/9781118445112.stat07841"},{"key":"13_CR2","doi-asserted-by":"publisher","unstructured":"Apruzzese, G., Laskov, P., Tastemirova, A.: SoK: the impact of unlabelled data in cyberthreat detection, May 2022. https:\/\/doi.org\/10.48550\/arXiv.2205.08944","DOI":"10.48550\/arXiv.2205.08944"},{"key":"13_CR3","doi-asserted-by":"publisher","unstructured":"Auer, F., Felderer, M.: Addressing data quality problems with metamorphic data relations. In: IEEE\/ACM 4th International Workshop on Metamorphic Testing (MET), pp. 76\u201383 (2019). https:\/\/doi.org\/10.1109\/MET.2019.00019","DOI":"10.1109\/MET.2019.00019"},{"issue":"1","key":"13_CR4","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s40537-021-00445-7","volume":"8","author":"FA Batarseh","year":"2021","unstructured":"Batarseh, F.A., Freeman, L., Huang, C.-H.: A survey on artificial intelligence assurance. J. Big Data 8(1), 1\u201330 (2021). https:\/\/doi.org\/10.1186\/s40537-021-00445-7","journal-title":"J. Big Data"},{"key":"13_CR5","doi-asserted-by":"publisher","unstructured":"Bergman, M., Milo, T., Novgorodov, S., Tan, W.C.: Query-oriented data cleaning with oracles. In: Proceedings of the 2015 ACM SIGMOD International Conference on Management of Data, SIGMOD 2015, pp. 1199\u20131214. Association for Computing Machinery, New York (2015). https:\/\/doi.org\/10.1145\/2723372.2737786","DOI":"10.1145\/2723372.2737786"},{"issue":"2","key":"13_CR6","doi-asserted-by":"publisher","first-page":"66","DOI":"10.1109\/MIC.2020.2979620","volume":"24","author":"S Bhatt","year":"2020","unstructured":"Bhatt, S., Sheth, A., Shalin, V., Zhao, J.: Knowledge graph semantic enhancement of input data for improving AI. IEEE Internet Comput. 24(2), 66\u201372 (2020). https:\/\/doi.org\/10.1109\/MIC.2020.2979620","journal-title":"IEEE Internet Comput."},{"key":"13_CR7","doi-asserted-by":"publisher","unstructured":"Caiafa, C.F., Zhe, S., Toshihisa, T., Pere, M.P., Sol\u00e9-Casals, J.: Machine learning methods with noisy, incomplete or small datasets. Appl. Sci. 11(9) (2021). https:\/\/doi.org\/10.3390\/app11094132","DOI":"10.3390\/app11094132"},{"key":"13_CR8","doi-asserted-by":"publisher","unstructured":"Camacho, J., Wasielewska, K.: Dataset quality assessment in autonomous networks with permutation testing. In: IEEE\/IFIP Network Operations and Management Symposium (NOMS), pp. 1\u20134 (2022). https:\/\/doi.org\/10.1109\/NOMS54207.2022.9789767","DOI":"10.1109\/NOMS54207.2022.9789767"},{"key":"13_CR9","doi-asserted-by":"publisher","first-page":"5371","DOI":"10.1109\/ACCESS.2020.3048319","volume":"9","author":"L Caviglione","year":"2021","unstructured":"Caviglione, L., et al.: Tight arms race: overview of current malware threats and trends in their detection. IEEE Access 9, 5371\u20135396 (2021). https:\/\/doi.org\/10.1109\/ACCESS.2020.3048319","journal-title":"IEEE Access"},{"key":"13_CR10","doi-asserted-by":"publisher","unstructured":"Cordeiro, F.R., Carneiro, G.: A survey on deep learning with noisy labels: how to train your model when you cannot trust on the annotations? In: 33rd SIBGRAPI Conference on Graphics, Patterns and Images (SIBGRAPI), pp. 9\u201316 (2020). https:\/\/doi.org\/10.1109\/SIBGRAPI51738.2020.00010","DOI":"10.1109\/SIBGRAPI51738.2020.00010"},{"key":"13_CR11","doi-asserted-by":"publisher","unstructured":"Ding, J., Li, X.: An approach for validating quality of datasets for machine learning. In: 2018 IEEE International Conference on Big Data (Big Data), pp. 2795\u20132803 (2018). https:\/\/doi.org\/10.1109\/BigData.2018.8622640","DOI":"10.1109\/BigData.2018.8622640"},{"key":"13_CR12","doi-asserted-by":"publisher","first-page":"115297","DOI":"10.1016\/j.eswa.2021.115297","volume":"182","author":"M Dudjak","year":"2021","unstructured":"Dudjak, M., Martinovic, G.: An empirical study of data intrinsic characteristics that make learning from imbalanced data difficult. Expert Syst. Appl. 182, 115297 (2021)","journal-title":"Expert Syst. Appl."},{"key":"13_CR13","doi-asserted-by":"publisher","first-page":"165263","DOI":"10.1109\/ACCESS.2020.3022633","volume":"8","author":"MS Elsayed","year":"2020","unstructured":"Elsayed, M.S., Le-Khac, N.A., Jurcut, A.D.: InSDN: a novel SDN intrusion dataset. IEEE Access 8, 165263\u2013165284 (2020). https:\/\/doi.org\/10.1109\/ACCESS.2020.3022633","journal-title":"IEEE Access"},{"key":"13_CR14","doi-asserted-by":"publisher","unstructured":"Engelen, G., Rimmer, V., Joosen, W.: Troubleshooting an intrusion detection dataset: the CICIDS2017 case study. In: 2021 IEEE Security and Privacy Workshops (SPW) (2021). https:\/\/doi.org\/10.1109\/SPW53761.2021.00009","DOI":"10.1109\/SPW53761.2021.00009"},{"key":"13_CR15","doi-asserted-by":"publisher","unstructured":"Gupta, S., Gupta, A.: Dealing with noise problem in machine learning data-sets: a systematic review. Procedia Comput. Sci. 161, 466\u2013474 (2019). https:\/\/doi.org\/10.1016\/j.procs.2019.11.146. 5th Information Systems International Conference, Surabaya, Indonesia","DOI":"10.1016\/j.procs.2019.11.146"},{"key":"13_CR16","doi-asserted-by":"publisher","first-page":"25","DOI":"10.54623\/fue.fcij.6.1.3","volume":"6","author":"M Ibrahim","year":"2021","unstructured":"Ibrahim, M., Helmy, Y., Elzanfaly, D.: Data quality dimensions, metrics, and improvement techniques. Future Comput. Inform. J. 6, 25\u201344 (2021). https:\/\/doi.org\/10.54623\/fue.fcij.6.1.3","journal-title":"Future Comput. Inform. J."},{"key":"13_CR17","doi-asserted-by":"publisher","unstructured":"Joyce, R.J., Raff, E., Nicholas, C.: A framework for cluster and classifier evaluation in the absence of reference labels. In: AISec 2021. Association for Computing Machinery, New York (2021). https:\/\/doi.org\/10.1145\/3474369.3486867","DOI":"10.1145\/3474369.3486867"},{"issue":"5\u20136","key":"13_CR18","doi-asserted-by":"publisher","first-page":"521","DOI":"10.1002\/(SICI)1099-128X(199609)10:5\/6<521::AID-CEM448>3.0.CO;2-J","volume":"10","author":"F Lindgren","year":"1996","unstructured":"Lindgren, F., Hansen, B., Karcher, W., Sj\u00f6str\u00f6m, M., Eriksson, L.: Model validation by permutation tests: applications to variable selection. J. Chemometr. 10(5\u20136), 521\u2013532 (1996)","journal-title":"J. Chemometr."},{"issue":"14","key":"13_CR19","doi-asserted-by":"publisher","first-page":"2242","DOI":"10.1021\/ac50064a004","volume":"52","author":"D MacDougall","year":"1980","unstructured":"MacDougall, D., Crummett, W.B.: Guidelines for data acquisition and data quality evaluation in environmental chemistry. Anal. Chem. 52(14), 2242\u20132249 (1980). https:\/\/doi.org\/10.1021\/ac50064a004","journal-title":"Anal. Chem."},{"key":"13_CR20","doi-asserted-by":"publisher","first-page":"411","DOI":"10.1016\/j.cose.2017.11.004","volume":"73","author":"G Maci\u00e1-Fern\u00e1ndez","year":"2018","unstructured":"Maci\u00e1-Fern\u00e1ndez, G., Camacho, J., Mag\u00e1n-Carri\u00f3n, R., Garc\u00eda-Teodoro, P., Ther\u00f3n, R.: UGR\u201916: a new dataset for the evaluation of cyclostationarity-based network IDSs. Comput. Secur. 73, 411\u2013424 (2018). https:\/\/doi.org\/10.1016\/j.cose.2017.11.004","journal-title":"Comput. Secur."},{"key":"13_CR21","first-page":"1833","volume":"11","author":"M Ojala","year":"2010","unstructured":"Ojala, M., Garriga, G.: Permutation tests for studying classifier performance. J. Mach. Learn. Res. 11, 1833\u20131863 (2010)","journal-title":"J. Mach. Learn. Res."},{"issue":"4","key":"13_CR22","first-page":"481","volume":"70","author":"F Pesarin","year":"2010","unstructured":"Pesarin, F., Salmaso, L.: The permutation testing approach: a review. Statistica (Bologna) 70(4), 481\u2013509 (2010)","journal-title":"Statistica (Bologna)"},{"issue":"2","key":"13_CR23","doi-asserted-by":"publisher","first-page":"639","DOI":"10.1007\/s11222-011-9261-0","volume":"22","author":"F Pesarin","year":"2012","unstructured":"Pesarin, F., Salmaso, L.: A review and some new results on permutation testing for multivariate problems. Stat. Comput. 22(2), 639\u2013646 (2012). https:\/\/doi.org\/10.1007\/s11222-011-9261-0","journal-title":"Stat. Comput."},{"key":"13_CR24","doi-asserted-by":"publisher","unstructured":"Pin, K., Kim, J.Y., Chang, J.H., Nam, Y.: Quality evaluation of fundus images using transfer learning. In: International Conference on Computational Science and Computational Intelligence (CSCI), pp. 742\u2013744 (2020). https:\/\/doi.org\/10.1109\/CSCI51800.2020.00139","DOI":"10.1109\/CSCI51800.2020.00139"},{"key":"13_CR25","doi-asserted-by":"publisher","unstructured":"Sahu, A., Mao, Z., Davis, K., Goulart, A.E.: Data processing and model selection for machine learning-based network intrusion detection. In: 2020 IEEE International Workshop Technical Committee on Communications Quality and Reliability (CQR), pp. 1\u20136 (2020). https:\/\/doi.org\/10.1109\/CQR47547.2020.9101394","DOI":"10.1109\/CQR47547.2020.9101394"},{"issue":"1","key":"13_CR26","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s40537-020-00318-5","volume":"7","author":"IH Sarker","year":"2020","unstructured":"Sarker, I.H., Kayes, A.S.M., Badsha, S., Alqahtani, H., Watters, P., Ng, A.: Cybersecurity data science: an overview from machine learning perspective. J. Big Data 7(1), 1\u201329 (2020). https:\/\/doi.org\/10.1186\/s40537-020-00318-5","journal-title":"J. Big Data"},{"key":"13_CR27","doi-asserted-by":"publisher","first-page":"222310","DOI":"10.1109\/ACCESS.2020.3041951","volume":"8","author":"K Shaukat","year":"2020","unstructured":"Shaukat, K., Luo, S., Varadharajan, V., Hameed, I.A., Xu, M.: A survey on machine learning techniques for cyber security in the last decade. IEEE Access 8, 222310\u2013222354 (2020). https:\/\/doi.org\/10.1109\/ACCESS.2020.3041951","journal-title":"IEEE Access"},{"key":"13_CR28","doi-asserted-by":"publisher","unstructured":"Soukup, D., Tisov\u010d\u00edk, P., Hynek, K., \u010cejka, T.: Towards evaluating quality of datasets for network traffic domain. In: 17th International Conference on Network and Service Management (CNSM), pp. 264\u2013268 (2021). https:\/\/doi.org\/10.23919\/CNSM52442.2021.9615601","DOI":"10.23919\/CNSM52442.2021.9615601"},{"key":"13_CR29","doi-asserted-by":"publisher","first-page":"107219","DOI":"10.1016\/j.asoc.2021.107219","volume":"104","author":"K Stapor","year":"2021","unstructured":"Stapor, K., Ksieniewicz, P., Garc\u00eda, S., Wo\u017aniak, M.: How to design the fair experimental classifier evaluation. Appl. Soft Comput. 104, 107219 (2021). https:\/\/doi.org\/10.1016\/j.asoc.2021.107219","journal-title":"Appl. Soft Comput."},{"key":"13_CR30","doi-asserted-by":"publisher","unstructured":"Taleb, I., El Kassabi, H., Serhani, M., Dssouli, R., Bouhaddioui, C.: Big data quality: a quality dimensions evaluation, July 2016. https:\/\/doi.org\/10.1109\/UIC-ATC-ScalCom-CBDCom-IoP-SmartWorld.2016.0122","DOI":"10.1109\/UIC-ATC-ScalCom-CBDCom-IoP-SmartWorld.2016.0122"},{"key":"13_CR31","doi-asserted-by":"publisher","unstructured":"Udeshi, S., Jiang, X., Chattopadhyay, S.: Callisto: entropy-based test generation and data quality assessment for machine learning systems. In: IEEE 13th International Conference on Software Testing, Validation and Verification (ICST), Los Alamitos, CA, USA, pp. 448\u2013453. IEEE Computer Society, October 2020. https:\/\/doi.org\/10.1109\/ICST46399.2020.00060","DOI":"10.1109\/ICST46399.2020.00060"},{"key":"13_CR32","doi-asserted-by":"publisher","unstructured":"Wasielewska, K., Soukup, D., \u010cejka, T., Camacho, J.: Dataset quality assessment with permutation testing showcased on network traffic datasets, June 2022. https:\/\/doi.org\/10.36227\/techrxiv.20145539.v1","DOI":"10.36227\/techrxiv.20145539.v1"},{"issue":"5","key":"13_CR33","doi-asserted-by":"publisher","first-page":"1179","DOI":"10.1007\/s10618-018-0554-1","volume":"32","author":"GI Webb","year":"2018","unstructured":"Webb, G.I., Lee, L.K., Goethals, B., Petitjean, F.: Analyzing concept drift and shift from sample data. Data Min. Knowl. Disc. 32(5), 1179\u20131199 (2018). https:\/\/doi.org\/10.1007\/s10618-018-0554-1","journal-title":"Data Min. Knowl. Disc."},{"issue":"1","key":"13_CR34","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s13638-020-01697-2","volume":"2020","author":"M Wu","year":"2020","unstructured":"Wu, M., et al.: Learning deep networks with crowdsourcing for relevance evaluation. EURASIP J. Wirel. Commun. Netw. 2020(1), 1\u201311 (2020). https:\/\/doi.org\/10.1186\/s13638-020-01697-2","journal-title":"EURASIP J. Wirel. Commun. Netw."},{"key":"13_CR35","unstructured":"Yoon, J., Arik, S., Pfister, T.: Data valuation using reinforcement learning. In Proceedings of the 37th International Conference on Machine Learning. Proceedings of Machine Learning Research, vol. 119, pp. 10842\u201310851 (2020). https:\/\/proceedings.mlr.press\/v119\/yoon20a.html"},{"issue":"3","key":"13_CR36","doi-asserted-by":"publisher","first-page":"264","DOI":"10.1109\/TSE.2015.2478001","volume":"42","author":"ZQ Zhou","year":"2016","unstructured":"Zhou, Z.Q., Xiang, S., Chen, T.Y.: Metamorphic testing for software quality assessment: a study of search engines. IEEE Trans. Softw. Eng. 42(3), 264\u2013284 (2016). https:\/\/doi.org\/10.1109\/TSE.2015.2478001","journal-title":"IEEE Trans. Softw. Eng."}],"container-title":["Communications in Computer and Information Science","Machine Learning and Principles and Practice of Knowledge Discovery in Databases"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-23633-4_13","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,30]],"date-time":"2023-01-30T07:21:34Z","timestamp":1675063294000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-23633-4_13"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031236327","9783031236334"],"references-count":36,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-23633-4_13","relation":{},"ISSN":["1865-0929","1865-0937"],"issn-type":[{"value":"1865-0929","type":"print"},{"value":"1865-0937","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"31 January 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECML PKDD","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Joint European Conference on Machine Learning and Knowledge Discovery in Databases","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Grenoble","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"France","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"19 September 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"23 September 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"22","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecml2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/2022.ecmlpkdd.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"CMT","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"1060","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"236","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"22% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3-4","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3-4","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"17 demo track papers have been accepted from 28 submissions","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}