{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,21]],"date-time":"2025-11-21T06:17:36Z","timestamp":1763705856487,"version":"3.40.3"},"publisher-location":"Cham","reference-count":25,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030587925"},{"type":"electronic","value":"9783030587932"}],"license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.1007\/978-3-030-58793-2_2","type":"book-chapter","created":{"date-parts":[[2020,8,30]],"date-time":"2020-08-30T23:04:02Z","timestamp":1598828642000},"page":"17-31","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":41,"title":["Towards Guidelines for Assessing Qualities of Machine Learning Systems"],"prefix":"10.1007","author":[{"given":"Julien","family":"Siebert","sequence":"first","affiliation":[]},{"given":"Lisa","family":"Joeckel","sequence":"additional","affiliation":[]},{"given":"Jens","family":"Heidrich","sequence":"additional","affiliation":[]},{"given":"Koji","family":"Nakamichi","sequence":"additional","affiliation":[]},{"given":"Kyoko","family":"Ohashi","sequence":"additional","affiliation":[]},{"given":"Isao","family":"Namba","sequence":"additional","affiliation":[]},{"given":"Rieko","family":"Yamamoto","sequence":"additional","affiliation":[]},{"given":"Mikio","family":"Aoyama","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2020,8,31]]},"reference":[{"key":"2_CR1","doi-asserted-by":"crossref","unstructured":"Wan, Z., Xia, X., Lo, D., Murphy, G.C.: How does machine learning change software development practices? IEEE Trans. Softw. Eng. 1 (2019)","DOI":"10.1109\/TSE.2019.2937083"},{"key":"2_CR2","unstructured":"Sculley, D., et al.: Hidden technical debt in machine learning systems. In: Proceedings of the 28th International Conference on Neural Information Processing Systems, pp. 2503\u20132511 (2015)"},{"key":"2_CR3","doi-asserted-by":"crossref","unstructured":"Zhang, J.M., Harman, M., Ma, L., Liu, Y.: Machine learning testing: survey, landscapes and horizons. IEEE Trans. Softw. Eng. 1 (2020)","DOI":"10.1109\/TSE.2019.2962027"},{"key":"2_CR4","unstructured":"ISO\/IEC 25010:2011: Systems and software engineering\u2014Systems and software Quality Requirements and Evaluation (SQuaRE)\u2014System and software quality models"},{"key":"2_CR5","unstructured":"ISO\/TS 8000:2011: Data Quality"},{"key":"2_CR6","unstructured":"High-Level Expert Group on Artificial Intelligence: Ethics Guidelines for Trustworthy AI. European Commission (2019)"},{"key":"2_CR7","unstructured":"DIN SPEC 92001-01: K\u00fcnstliche Intelligenz - Life Cycle Prozesse und Qualit\u00e4tsanforderungen. Teil 1: Qualit\u00e4ts-Meta-Modell. Beuth Verlag GmbH, Berlin"},{"key":"2_CR8","unstructured":"Hamada, K., Ishikawa, F., Masuda, S., Matsuya, M., Ujita, Y.: Guidelines for quality assurance of machine learning-based artificial intelligence. In: SEKE2020: the 32nd International Conference on Software Engineering & Knowledge Engineering, pp. 335\u2013341 (2020)"},{"key":"2_CR9","unstructured":"Trustworthy Use of Artificial Intelligence. Priorities from a Philosophical, Ethical, Legal, and Technological Viewpoint as a Basis for Certification of Artificial Intelligence. Fraunhofer Institute for Intelligent Analysis and Information Systems IAIS Schloss Birlinghoven (2019)"},{"key":"2_CR10","unstructured":"From Principles to Practice. An interdisciplinary framework to operationalise AI ethics. VDE, Bertelsmann Stiftung (2020)"},{"key":"2_CR11","volume-title":"Testing of Artificial Intelligence","author":"R Marselis","year":"2017","unstructured":"Marselis, R., Shaukat, H., Gansel, T.: Testing of Artificial Intelligence. Sogeti, Paris (2017)"},{"key":"2_CR12","volume-title":"Machine Intelligence Quality Characteristics. How to Measure the Quality of Artificial Intelligence and Robotics","author":"R Marselis","year":"2018","unstructured":"Marselis, R., Shaukat, H.: Machine Intelligence Quality Characteristics. How to Measure the Quality of Artificial Intelligence and Robotics. Sogeti, Paris (2018)"},{"key":"2_CR13","doi-asserted-by":"crossref","unstructured":"Nakajima, S.: Quality assurance of machine learning software. In: 2018 IEEE 7th Global Conference on Consumer Electronics (GCCE), 9\u201312 October 2018, pp. 601\u2013604. IEEE, Piscataway (2018)","DOI":"10.1109\/GCCE.2018.8574766"},{"key":"2_CR14","doi-asserted-by":"publisher","first-page":"137","DOI":"10.1017\/S0269888910000032","volume":"25","author":"G Mariscal","year":"2010","unstructured":"Mariscal, G., Marb\u00e1n, \u00d3., Fern\u00e1ndez, C.: A survey of data mining and knowledge discovery process models and methodologies. Knowl. Eng. Rev. 25, 137\u2013166 (2010)","journal-title":"Knowl. Eng. Rev."},{"key":"2_CR15","doi-asserted-by":"crossref","unstructured":"Martinez-Plumed, F., et al.: CRISP-DM twenty years later: from data mining processes to data science trajectories. IEEE Trans. Knowl. Data Eng. 1 (2020)","DOI":"10.1109\/TKDE.2019.2962680"},{"key":"2_CR16","series-title":"Lecture Notes in Business Information Processing","doi-asserted-by":"publisher","first-page":"227","DOI":"10.1007\/978-3-030-19034-7_14","volume-title":"Agile Processes in Software Engineering and Extreme Programming","author":"LE Lwakatare","year":"2019","unstructured":"Lwakatare, L.E., Raj, A., Bosch, J., Olsson, H.H., Crnkovic, I.: A taxonomy of software engineering challenges for machine learning systems: an empirical investigation. In: Kruchten, P., Fraser, S., Coallier, F. (eds.) XP 2019. LNBIP, vol. 355, pp. 227\u2013243. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-19034-7_14"},{"key":"2_CR17","doi-asserted-by":"crossref","unstructured":"Amershi, S., et al.: Software engineering for machine learning: a case study. In: 2019 IEEE\/ACM 41st International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP), pp. 291\u2013300 (2019)","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"2_CR18","first-page":"1","volume":"5","author":"M Hossin","year":"2015","unstructured":"Hossin, M., Sulaiman, M.N.: A review on evaluation metrics for data classification evaluations. IJDKP 5, 1\u201311 (2015)","journal-title":"IJDKP"},{"key":"2_CR19","doi-asserted-by":"publisher","first-page":"e0159161","DOI":"10.1371\/journal.pone.0159161","volume":"11","author":"S Emmons","year":"2016","unstructured":"Emmons, S., Kobourov, S., Gallant, M., B\u00f6rner, K.: Analysis of network clustering algorithms and cluster quality metrics at scale. PLoS ONE 11, e0159161 (2016)","journal-title":"PLoS ONE"},{"key":"2_CR20","doi-asserted-by":"publisher","first-page":"23","DOI":"10.1145\/3144172","volume":"60","author":"S Barocas","year":"2017","unstructured":"Barocas, S., Boyd, D.: Engaging the ethics of data science in practice. Commun. ACM 60, 23\u201325 (2017)","journal-title":"Commun. ACM"},{"key":"2_CR21","unstructured":"Kleinberg, J., Mullainathan, S., Raghavan, M.: Inherent Trade-Offs in the Fair Determination of Risk Scores. arXiv.org (2016)"},{"key":"2_CR22","doi-asserted-by":"publisher","first-page":"101","DOI":"10.1016\/j.infsof.2015.02.009","volume":"62","author":"S Wagner","year":"2015","unstructured":"Wagner, S., et al.: Operationalised product quality models and assessment: the Quamoco approach. Inf. Softw. Technol. 62, 101\u2013123 (2015)","journal-title":"Inf. Softw. Technol."},{"key":"2_CR23","doi-asserted-by":"crossref","unstructured":"Kaufman, S., Rosset, S., Perlich, C.: Leakage in data mining. In: Apte, C., Ghosh, J., Smyth, P. (eds.) Proceedings of the 17th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, San Diego, Ca, USA, 21\u201324 August 2011, p. 556. ACM, New York (2011)","DOI":"10.1145\/2020408.2020496"},{"key":"2_CR24","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"431","DOI":"10.1007\/978-3-319-99229-7_36","volume-title":"Computer Safety, Reliability, and Security","author":"M Kl\u00e4s","year":"2018","unstructured":"Kl\u00e4s, M., Vollmer, A.M.: Uncertainty in machine learning applications: a practice-driven classification of uncertainty. In: Gallina, B., Skavhaug, A., Schoitsch, E., Bitsch, F. (eds.) SAFECOMP 2018. LNCS, vol. 11094, pp. 431\u2013438. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-319-99229-7_36"},{"key":"2_CR25","unstructured":"Nakamichi, K., et al.: Requirements-driven method to determine quality characteristics and measurements for machine learning software and its evaluation. In: 28th IEEE International Requirements Engineering Conference (RE\u201920)"}],"container-title":["Communications in Computer and Information Science","Quality of Information and Communications Technology"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-58793-2_2","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,4,24]],"date-time":"2021-04-24T08:28:42Z","timestamp":1619252922000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-58793-2_2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"ISBN":["9783030587925","9783030587932"],"references-count":25,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-58793-2_2","relation":{},"ISSN":["1865-0929","1865-0937"],"issn-type":[{"type":"print","value":"1865-0929"},{"type":"electronic","value":"1865-0937"}],"subject":[],"published":{"date-parts":[[2020]]},"assertion":[{"value":"31 August 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"QUATIC","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on the Quality of Information and Communications Technology","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Faro","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Portugal","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2020","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"9 September 2020","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"11 September 2020","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"13","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"quatic2020","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/2020.quatic.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"81","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"27","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"12","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"33% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"The conference was held virtually due to the COVID-19 pandemic.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}