{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,10]],"date-time":"2025-09-10T22:23:26Z","timestamp":1757543006213,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":51,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,16]],"date-time":"2024-04-16T00:00:00Z","timestamp":1713225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,16]]},"DOI":"10.1145\/3643691.3648585","type":"proceedings-article","created":{"date-parts":[[2024,7,29]],"date-time":"2024-07-29T14:29:09Z","timestamp":1722263349000},"page":"20-27","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["PARMA: a Platform Architecture to enable Automated, Reproducible, and Multi-party Assessments of AI Trustworthiness"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0007-6663-2368","authenticated-orcid":false,"given":"Maximilian","family":"Pintz","sequence":"first","affiliation":[{"name":"Fraunhofer IAIS, University of Bonn, Sankt Augustin, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-3247-9222","authenticated-orcid":false,"given":"Daniel","family":"Becker","sequence":"additional","affiliation":[{"name":"Fraunhofer IAIS, Sankt Augustin, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7352-9165","authenticated-orcid":false,"given":"Michael","family":"Mock","sequence":"additional","affiliation":[{"name":"Fraunhofer IAIS, Sankt Augustin, Germany"}]}],"member":"320","published-online":{"date-parts":[[2024,7,29]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/IEEESTD.2021.9325968"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1109\/IEEESTD.2022.9698145"},{"key":"e_1_3_2_1_3_1","volume-title":"what is ai verify?","author":"Ai","year":"2023","unstructured":"Ai verify foundation. what is ai verify? 2023."},{"key":"e_1_3_2_1_4_1","volume-title":"Information technology - Artificial intelligence - Guidance on risk management. Standard","author":"IEC","year":"2023","unstructured":"ISO\/IEC 23894:2023. Information technology - Artificial intelligence - Guidance on risk management. Standard, International Organization for Standardization, Geneva, CH, 2023. URL https:\/\/www.iso.org\/standard\/77304.html."},{"key":"e_1_3_2_1_5_1","volume-title":"Proceedings of CIDR","volume":"8","author":"Armbrust Michael","year":"2021","unstructured":"Michael Armbrust, Ali Ghodsi, Reynold Xin, et al. Lakehouse: a new generation of open platforms that unify data warehousing and advanced analytics. In Proceedings of CIDR, volume 8, 2021."},{"issue":"26","key":"e_1_3_2_1_6_1","first-page":"353","article-title":"Reproducibility crisis","volume":"533","author":"Baker Monya","year":"2016","unstructured":"Monya Baker. Reproducibility crisis. Nature, 533(26):353--66, 2016.","journal-title":"Nature"},{"volume-title":"Test Driven Development: By Example","year":"2002","key":"e_1_3_2_1_7_1","unstructured":"Beck. Test Driven Development: By Example. Addison-Wesley Longman Publishing Co., Inc., USA, 2002. ISBN 0321146530."},{"key":"e_1_3_2_1_8_1","volume-title":"Zukunftssichere L\u00f6sungen f\u00fcr maschinelles Lernen","author":"Beck Niklas","year":"2020","unstructured":"Niklas Beck, Claudio Martens, Karl-Heinz Sylla, et al. Zukunftssichere L\u00f6sungen f\u00fcr maschinelles Lernen. 2020."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533090"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1147\/JRD.2019.2942287"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2020.110542"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.mlwa.2021.100134"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/EuroSP.2019.00043"},{"key":"e_1_3_2_1_14_1","volume-title":"Proposal for a regulation laying down harmonised rules on artificial int certain union legislative acts","author":"European Commission","year":"2021","unstructured":"European Commission. Proposal for a regulation laying down harmonised rules on artificial int certain union legislative acts, 2021."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","DOI":"10.1017\/S0956796810000195","article-title":"A purely functional linux distribution","author":"Dolstra Eelco","year":"2010","unstructured":"Eelco Dolstra, Andres L\u00f6h, and Nicolas Pierron. Nixos: A purely functional linux distribution. Journal of Functional Programming, 20(5--6):577--615, 2010.","journal-title":"Journal of Functional Programming, 20(5--6):577--615"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3493700.3493768"},{"key":"e_1_3_2_1_17_1","volume-title":"Foundations of Software Testing: ISTQB Certification. Course Technology Cengage Learning","author":"Graham D.","year":"2008","unstructured":"D. Graham. Foundations of Software Testing: ISTQB Certification. Course Technology Cengage Learning, 2008. ISBN 9781844809899."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2021.06.014"},{"key":"e_1_3_2_1_19_1","first-page":"73","volume-title":"EuroVis Workshop on Visual Analytics","author":"Haedecke Elena","unstructured":"Elena Haedecke, Michael Mock, and Maram Akila. Scrutinai: a visual analytics approach for the semantic analysis of deep neural network predictions. In EuroVis Workshop on Visual Analytics (EuroVA), pp. 73--775. The Eurographics Association, 2022."},{"key":"e_1_3_2_1_20_1","volume-title":"The ethics of AI ethics - an evaluation of guidelines. CoRR, abs\/1903.03425","author":"Hagendorff Thilo","year":"2019","unstructured":"Thilo Hagendorff. The ethics of AI ethics - an evaluation of guidelines. CoRR, abs\/1903.03425, 2019. URL http:\/\/arxiv.org\/abs\/1903.03425."},{"issue":"34","key":"e_1_3_2_1_21_1","first-page":"1","article-title":"Quantus: An explainable ai toolkit for responsible evaluation of neural network explanations and beyond","volume":"24","author":"Hedstr\u00f6m Anna","year":"2023","unstructured":"Anna Hedstr\u00f6m, Leander Weber, Daniel Krakowczyk, et al. Quantus: An explainable ai toolkit for responsible evaluation of neural network explanations and beyond. Journal of Machine Learning Research, 24(34):1--11, 2023.","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_1_22_1","volume-title":"2024 International Workshop on Responsible AI Engineering (RAIE '24)","author":"Helmer Lennard","year":"2024","unstructured":"Lennard Helmer, Claudio Martens, Dennis Wegener, et al. Towards Trustworthy AI Engineering - A Case study on integrating an AI audit catalogue into MLOps processes. In 2024 International Workshop on Responsible AI Engineering (RAIE '24), 2024."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-01233-4_1"},{"key":"e_1_3_2_1_24_1","volume-title":"Open container initiative runtime specification","author":"Initiative Open Containers","year":"2020","unstructured":"Open Containers Initiative et al. Open container initiative runtime specification, 2020."},{"issue":"01","key":"e_1_3_2_1_25_1","doi-asserted-by":"crossref","first-page":"83","DOI":"10.1142\/S2424862221300040","article-title":"Artificial intelligence applications for industry 4.0: A literature-based study","volume":"7","author":"Javaid Mohd","year":"2022","unstructured":"Mohd Javaid, Abid Haleem, Ravi Pratap Singh, et al. Artificial intelligence applications for industry 4.0: A literature-based study. Journal of Industrial Integration and Management, 7(01):83--111, 2022.","journal-title":"Journal of Industrial Integration and Management"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2018.3571224"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3569928"},{"key":"e_1_3_2_1_28_1","volume-title":"Transactions on Machine Learning Research","author":"Liang Percy","year":"2023","unstructured":"Percy Liang, Rishi Bommasani, Tony Lee, et al. Holistic evaluation of language models. Transactions on Machine Learning Research, 2023. ISSN 2835-8856. Featured Certification, Expert Certification."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3522664.3528607"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626234"},{"key":"e_1_3_2_1_31_1","first-page":"26230","article-title":"On the tradeoff between robustness and fairness","volume":"35","author":"Ma Xinsong","year":"2022","unstructured":"Xinsong Ma, Zekai Wang, and Weiwei Liu. On the tradeoff between robustness and fairness. Advances in Neural Information Processing Systems, 35:26230--26241, 2022.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_32_1","first-page":"15458","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence","volume":"35","author":"McGregor Sean","year":"2021","unstructured":"Sean McGregor. Preventing repeated real world ai failures by cataloging incidents: The ai incident database. In Proceedings of the AAAI Conference on Artificial Intelligence, volume 35, pp. 15458--15463, 2021."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2851613.2851737"},{"key":"e_1_3_2_1_34_1","volume-title":"NCSC. Guidelines for secure AI system development. Report","author":"CISA","year":"2023","unstructured":"CISA et al. NCSC. Guidelines for secure AI system development. Report, National Cyber Security Centre, Cybersecurity and Infrastructure Security Agency, London, GB, 2023. URL https:\/\/www.ncsc.gov.uk\/files\/Guidelines-for-secure-AI-system-development.pdf."},{"key":"e_1_3_2_1_35_1","volume-title":"Minh Ngoc Tran, et al. Adversarial robustness toolbox v1.0.0","author":"Nicolae Maria-Irina","year":"2019","unstructured":"Maria-Irina Nicolae, Mathieu Sinn, Minh Ngoc Tran, et al. Adversarial robustness toolbox v1.0.0, 2019."},{"key":"e_1_3_2_1_36_1","volume-title":"European Commission","author":"High-Level Expert Group on Artificial Intelligence.","year":"2019","unstructured":"High-Level Expert Group on Artificial Intelligence. Ethics Guidelines for Trustworthy AI. Report, European Commission, Brussels, B, 2019. URL https:\/\/digital-strategy.ec.europa.eu\/en\/library\/ethics-guidelines-trustworthy-ai."},{"key":"e_1_3_2_1_37_1","volume-title":"ML Evaluation Standards Workshop at ICLR 2022","author":"Pintz Maximilian","year":"2022","unstructured":"Maximilian Pintz, Joachim Sicking, Maximilian Poretschkin, et al. A survey on uncertainty toolkits for deep learning. ML Evaluation Standards Workshop at ICLR 2022, 2022."},{"key":"e_1_3_2_1_38_1","volume-title":"Guideline for Trustworthy Artificial Intelligence - AI Assessment Catalog","author":"Poretschkin Maximilian","year":"2023","unstructured":"Maximilian Poretschkin, Anna Schmitz, Maram Akila, et al. Guideline for Trustworthy Artificial Intelligence - AI Assessment Catalog, 2023. URL https:\/\/arxiv.org\/abs\/2307.03681."},{"key":"e_1_3_2_1_39_1","volume-title":"Towards an interoperable ecosystem of ai and lt platforms: A roadmap for the implementation of different levels of interoperability. arXiv preprint arXiv:2004.08355","author":"Rehm Georg","year":"2020","unstructured":"Georg Rehm, Dimitrios Galanis, Penny Labropoulou, et al. Towards an interoperable ecosystem of ai and lt platforms: A roadmap for the implementation of different levels of interoperability. arXiv preprint arXiv:2004.08355, 2020."},{"key":"e_1_3_2_1_40_1","volume-title":"Auditing algorithms : Research methods for detecting discrimination on internet platforms","author":"Sandvig Christian","year":"2014","unstructured":"Christian Sandvig, Kevin Hamilton, Karrie Karahalios, et al. Auditing algorithms : Research methods for detecting discrimination on internet platforms. 2014. URL https:\/\/api.semanticscholar.org\/CorpusID:15686114."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1515\/auto-2022-0012"},{"key":"e_1_3_2_1_42_1","volume-title":"James Crowley, et al. Composing complex and hybrid ai solutions. arXiv preprint arXiv:2202.12566","author":"Sch\u00fcller Peter","year":"2022","unstructured":"Peter Sch\u00fcller, Jo\u00e3o Paolo Costeira, James Crowley, et al. Composing complex and hybrid ai solutions. arXiv preprint arXiv:2202.12566, 2022."},{"key":"e_1_3_2_1_43_1","volume-title":"Fairness as a service (faas): verifiable and privacy-preserving fairness auditing of machine learning systems jo - international journal of information security","author":"Toreini Ehsan","year":"2023","unstructured":"Ehsan Toreini, Maryam Mehrnezhad, and Aad van Moorsel. Fairness as a service (faas): verifiable and privacy-preserving fairness auditing of machine learning systems jo - international journal of information security. 2023. URL https:\/\/api.semanticscholar.org\/CorpusID:15686114."},{"key":"e_1_3_2_1_44_1","volume-title":"Natural language processing advancements by deep learning: A survey. arXiv preprint arXiv:2003.01200","author":"Torfi Amirsina","year":"2020","unstructured":"Amirsina Torfi, Rouzbeh A Shirvani, Yaser Keneshloo, et al. Natural language processing advancements by deep learning: A survey. arXiv preprint arXiv:2003.01200, 2020."},{"key":"e_1_3_2_1_45_1","volume-title":"International Conference on Learning Representations","author":"Tsipras Dimitris","year":"2018","unstructured":"Dimitris Tsipras, Shibani Santurkar, Logan Engstrom, et al. Robustness may be at odds with accuracy. In International Conference on Learning Representations, 2018."},{"key":"e_1_3_2_1_46_1","first-page":"64","article-title":"Evaluation of inter-process communication mechanisms","volume":"86","author":"Venkataraman Aditya","year":"2015","unstructured":"Aditya Venkataraman and Kishore Kumar Jagadeesha. Evaluation of inter-process communication mechanisms. Architecture, 86:64, 2015.","journal-title":"Architecture"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/2641562"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/CAIN58948.2023.00027"},{"issue":"4","key":"e_1_3_2_1_49_1","first-page":"39","article-title":"Accelerating the machine learning lifecycle with mlflow","volume":"41","author":"Zaharia Matei","year":"2018","unstructured":"Matei Zaharia, Andrew Chen, Aaron Davidson, et al. Accelerating the machine learning lifecycle with mlflow. IEEE Data Eng. Bull., 41(4):39--45, 2018.","journal-title":"IEEE Data Eng. Bull."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2962027"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1007\/s40747-020-00212-w"}],"event":{"name":"RAIE '24: 2nd International Workshop on Responsible AI Engineering","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"],"location":"Lisbon Portugal","acronym":"RAIE '24"},"container-title":["Proceedings of the 2nd International Workshop on Responsible AI Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3643691.3648585","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3643691.3648585","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:04:20Z","timestamp":1750291460000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3643691.3648585"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,16]]},"references-count":51,"alternative-id":["10.1145\/3643691.3648585","10.1145\/3643691"],"URL":"https:\/\/doi.org\/10.1145\/3643691.3648585","relation":{},"subject":[],"published":{"date-parts":[[2024,4,16]]},"assertion":[{"value":"2024-07-29","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}