{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,31]],"date-time":"2026-07-31T09:25:25Z","timestamp":1785489925026,"version":"3.56.0"},"publisher-location":"Cham","reference-count":49,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031432637","type":"print"},{"value":"9783031432644","type":"electronic"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-43264-4_16","type":"book-chapter","created":{"date-parts":[[2023,9,6]],"date-time":"2023-09-06T23:03:02Z","timestamp":1694041382000},"page":"244-262","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["Robust Explanations for\u00a0Human-Neural Multi-agent Systems with\u00a0Formal Verification"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8245-9429","authenticated-orcid":false,"given":"Francesco","family":"Leofante","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3420-723X","authenticated-orcid":false,"given":"Alessio","family":"Lomuscio","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2023,9,7]]},"reference":[{"key":"16_CR1","doi-asserted-by":"crossref","unstructured":"Akintunde, M., Botoeva, E., Kouvaros, P., Lomuscio, A.: Formal verification of neural agents in non-deterministic environments. J. Auton. Agents Multi-Agent Syst. 36(1) (2022)","DOI":"10.1007\/s10458-021-09529-3"},{"key":"16_CR2","doi-asserted-by":"publisher","first-page":"132","DOI":"10.1016\/j.artint.2016.10.005","volume":"242","author":"S Barrett","year":"2017","unstructured":"Barrett, S., Rosenfeld, A., Kraus, S., Stone, P.: Making friends on the fly: cooperating with new teammates. Artif. Intell. 242, 132\u2013171 (2017)","journal-title":"Artif. Intell."},{"key":"16_CR3","doi-asserted-by":"crossref","unstructured":"Bj\u00f6rkegren, D., Blumenstock, J., Knight, S.: Manipulation-proof machine learning. arXiv preprint arXiv:2004.03865 (2020)","DOI":"10.1257\/rct.4649-1.0"},{"key":"16_CR4","unstructured":"Black, E., Wang, Z., Fredrikson, M.: Consistent counterfactuals for deep models. In: Proceedings of the International Conference on Learning Representations (ICLR22). OpenReview.net (2022)"},{"key":"16_CR5","doi-asserted-by":"crossref","unstructured":"Botoeva, E., Kouvaros, P., Kronqvist, J., Lomuscio, A., Misener, R.: Efficient verification of neural networks via dependency analysis. In: Proceedings of the 34th AAAI Conference on Artificial Intelligence (AAAI20), pp. 3291\u20133299. AAAI Press (2020)","DOI":"10.1609\/aaai.v34i04.5729"},{"key":"16_CR6","doi-asserted-by":"crossref","unstructured":"Brix, C., M\u00fcller, M.N., Bak, S., Johnson, T.T., Liu, C.: First three years of the international verification of neural networks competition (VNN-COMP). arXiv preprint arXiv:2301.05815 (2023)","DOI":"10.1007\/s10009-023-00703-4"},{"key":"16_CR7","doi-asserted-by":"crossref","unstructured":"Byrne, R.: Counterfactuals in explainable artificial intelligence (XAI): evidence from human reasoning. In: Proceedings of the 28th International Joint Conference on Artificial Intelligence, IJCAI19, pp. 6276\u20136282 (2019)","DOI":"10.24963\/ijcai.2019\/876"},{"key":"16_CR8","unstructured":"Dhurandhar, A., et al.: Explanations based on the missing: towards contrastive explanations with pertinent negatives. In: Advances in Neural Information Processing Systems (NeurIPS18), pp. 590\u2013601 (2018)"},{"key":"16_CR9","unstructured":"Dutta, S., Long, J., Mishra, S., Tilli, C., Magazzeni, D.: Robust counterfactual explanations for tree-based ensembles. In: Proceedings of the International Conference on Machine Learning (ICML22). Proceedings of Machine Learning Research, vol. 162, pp. 5742\u20135756. PMLR (2022)"},{"key":"16_CR10","unstructured":"FICO Community: Explainable Machine Learning Challenge (2019). https:\/\/community.fico.com\/s\/explainable-machine-learning-challenge"},{"key":"16_CR11","unstructured":"Guidotti, D., Leofante, F., Pulina, L., Tacchella, A.: Verification of neural networks: enhancing scalability through pruning. In: Proceedings of the 24th European Conference on Artificial Intelligence (ECAI20), pp. 2505\u20132512. IOS Press (2020)"},{"key":"16_CR12","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"357","DOI":"10.1007\/978-3-030-88885-5_23","volume-title":"Automated Technology for Verification and Analysis","author":"D Guidotti","year":"2021","unstructured":"Guidotti, D., Pulina, L., Tacchella, A.: pyNeVer: a framework for learning and\u00a0verification of neural networks. In: Hou, Z., Ganesh, V. (eds.) ATVA 2021. LNCS, vol. 12971, pp. 357\u2013363. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-88885-5_23"},{"key":"16_CR13","doi-asserted-by":"crossref","unstructured":"Hancox-Li, L.: Robustness in machine learning explanations: does it matter? In: Proceedings of the 2020 Conference on Fairness, Accountability, and Transparency (FAT*20), pp. 640\u2013647. ACM (2020)","DOI":"10.1145\/3351095.3372836"},{"key":"16_CR14","unstructured":"Henriksen, P., Hammernik, K., Rueckert, D., Lomuscio, A.: Bias field robustness verification of large neural image classifiers. In: Proceedings of the 32nd British Machine Vision Conference (BMVC21). BMVA Press (2021)"},{"key":"16_CR15","unstructured":"Henriksen, P., Lomuscio, A.: Efficient neural network verification via adaptive refinement and adversarial search. In: Proceedings of the 24th European Conference on Artificial Intelligence (ECAI20), pp. 2513\u20132520. IOS Press (2020)"},{"key":"16_CR16","doi-asserted-by":"crossref","unstructured":"Henriksen, P., Lomuscio, A.: DEEPSPLIT: an efficient splitting method for neural network verification via indirect effect analysis. In: Proceedings of the 30th International Joint Conference on Artificial Intelligence (IJCAI21), pp. 2549\u20132555. ijcai.org (2021)","DOI":"10.24963\/ijcai.2021\/351"},{"issue":"12","key":"16_CR17","doi-asserted-by":"publisher","first-page":"80","DOI":"10.1145\/2629559","volume":"57","author":"NR Jennings","year":"2014","unstructured":"Jennings, N.R., et al.: Human-agent collectives. Commun. ACM 57(12), 80\u201388 (2014)","journal-title":"Commun. ACM"},{"key":"16_CR18","doi-asserted-by":"crossref","unstructured":"Jiang, J., Leofante, F., Rago, A., Toni, F.: Formalising the robustness of counterfactual explanations for neural networks. In: Proceedings of the 37th AAAI Conference on Artificial Intelligence (AAAI23), pp. 14901\u201314909. AAAI Press (2023)","DOI":"10.1609\/aaai.v37i12.26740"},{"key":"16_CR19","doi-asserted-by":"crossref","unstructured":"Johnson, T., et al.: ARCH-COMP20 category report: artificial intelligence and neural network control systems (AINNCS) for continuous and hybrid systems plants. In: Proceedings of the 7th International Workshop on Applied Verification of Continuous and Hybrid Systems (ARCH20), pp. 107\u2013139. EasyChair (2020)","DOI":"10.29007\/9xgv"},{"key":"16_CR20","unstructured":"Karimi, A., Barthe, G., Balle, B., Valera, I.: Model-agnostic counterfactual explanations for consequential decisions. In: Proceedings of the 23rd International Conference on Artificial Intelligence and Statistics (AISTATS20), pp. 895\u2013905. PMLR (2020)"},{"key":"16_CR21","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"97","DOI":"10.1007\/978-3-319-63387-9_5","volume-title":"Computer Aided Verification","author":"G Katz","year":"2017","unstructured":"Katz, G., Barrett, C., Dill, D.L., Julian, K., Kochenderfer, M.J.: Reluplex: an efficient SMT solver for verifying deep neural networks. In: Majumdar, R., Kun\u010dak, V. (eds.) CAV 2017. LNCS, vol. 10426, pp. 97\u2013117. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-63387-9_5"},{"key":"16_CR22","doi-asserted-by":"crossref","unstructured":"Kenny, E., Keane, M.: On generating plausible counterfactual and semi-factual explanations for deep learning. In: Proceedings of the 35th AAAI Conference on Artificial Intelligence, AAAI21, pp. 11575\u201311585. AAAI Press (2021)","DOI":"10.1609\/aaai.v35i13.17377"},{"key":"16_CR23","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"730","DOI":"10.1007\/978-3-030-90870-6_41","volume-title":"Formal Methods","author":"P Kouvaros","year":"2021","unstructured":"Kouvaros, P., et al.: Formal analysis of neural network-based systems in the aircraft domain. In: Huisman, M., P\u0103s\u0103reanu, C., Zhan, N. (eds.) FM 2021. LNCS, vol. 13047, pp. 730\u2013740. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-90870-6_41"},{"issue":"7553","key":"16_CR24","doi-asserted-by":"publisher","first-page":"436","DOI":"10.1038\/nature14539","volume":"521","author":"Y LeCun","year":"2015","unstructured":"LeCun, Y., Bengio, Y., Hinton, G.: Deep learning. Nature 521(7553), 436\u2013444 (2015)","journal-title":"Nature"},{"key":"16_CR25","doi-asserted-by":"crossref","unstructured":"Leofante, F., Botoeva, E., Rajani, V.: Counterfactual explanations and model multiplicity: a relational verification view. In: Proceedings of the 20th International Conference on Principles of Knowledge Representation and Reasoning (KR23) (2023, to appear)","DOI":"10.24963\/kr.2023\/78"},{"key":"16_CR26","doi-asserted-by":"crossref","unstructured":"Leofante, F., Lomuscio, A.: Towards robust contrastive explanations for human-neural multi-agent systems. In: Proceedings of the International Conference on Autonomous Agents and Multiagent Systems (AAMAS23), pp. 2343\u20132345. ACM (2023)","DOI":"10.1007\/978-3-031-43264-4_16"},{"key":"16_CR27","unstructured":"Leofante, F., Narodytska, N., Pulina, L., Tacchella, A.: Automated verification of neural networks: advances, challenges and perspectives. CoRR abs\/1805.09938 (2018)"},{"issue":"3\u20134","key":"16_CR28","doi-asserted-by":"publisher","first-page":"244","DOI":"10.1561\/2400000035","volume":"4","author":"C Liu","year":"2021","unstructured":"Liu, C., Arnon, T., Lazarus, C., Strong, C.A., Barrett, C.W., Kochenderfer, M.J.: Algorithms for verifying deep neural networks. Found. Trends Optim. 4(3\u20134), 244\u2013404 (2021)","journal-title":"Found. Trends Optim."},{"key":"16_CR29","unstructured":"Lomuscio, A., Maganti, L.: An approach to reachability analysis for feed-forward ReLU neural networks. arXiv preprint arXiv:1706.07351 (2017)"},{"key":"16_CR30","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"650","DOI":"10.1007\/978-3-030-86520-7_40","volume-title":"Machine Learning and Knowledge Discovery in Databases. Research Track","author":"A Van Looveren","year":"2021","unstructured":"Van Looveren, A., Klaise, J.: Interpretable counterfactual explanations guided by prototypes. In: Oliver, N., P\u00e9rez-Cruz, F., Kramer, S., Read, J., Lozano, J.A. (eds.) ECML PKDD 2021. LNCS (LNAI), vol. 12976, pp. 650\u2013665. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-86520-7_40"},{"issue":"1","key":"16_CR31","doi-asserted-by":"publisher","first-page":"41","DOI":"10.1080\/13546780143000125","volume":"8","author":"R McCloy","year":"2002","unstructured":"McCloy, R., Byrne, R.: Semifactual \u201ceven if\u2019\u2019 thinking. Thinking Reason. 8(1), 41\u201367 (2002)","journal-title":"Thinking Reason."},{"key":"16_CR32","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.artint.2018.07.007","volume":"267","author":"T Miller","year":"2019","unstructured":"Miller, T.: Explanation in artificial intelligence: insights from the social sciences. Artif. Intell. 267, 1\u201338 (2019)","journal-title":"Artif. Intell."},{"key":"16_CR33","doi-asserted-by":"crossref","unstructured":"Mohammadi, K., Karimi, A., Barthe, G., Valera, I.: Scaling guarantees for nearest counterfactual explanations. In: Proceedings of the AAAI\/ACM Conference on AI, Ethics, and Society (AIES21), pp. 177\u2013187. ACM (2021)","DOI":"10.1145\/3461702.3462514"},{"key":"16_CR34","doi-asserted-by":"crossref","unstructured":"Mohapatra, J., Weng, T., Chen, P., Liu, S., Daniel, L.: Towards verifying robustness of neural networks against A family of semantic perturbations. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR20), pp. 241\u2013249. IEEE (2020)","DOI":"10.1109\/CVPR42600.2020.00032"},{"key":"16_CR35","doi-asserted-by":"crossref","unstructured":"Mothilal, R.K., Sharma, A., Tan, C.: Explaining machine learning classifiers through diverse counterfactual explanations. In: Proceedings of the International Conference on Fairness, Accountability, and Transparency (FAT*20), pp. 607\u2013617. ACM (2020)","DOI":"10.1145\/3351095.3372850"},{"key":"16_CR36","unstructured":"Pawelczyk, M., Agarwal, C., Joshi, S., Upadhyay, S., Lakkaraju, H.: Exploring counterfactual explanations through the lens of adversarial examples: a theoretical and empirical analysis. In: Proceedings of the International Conference on Artificial Intelligence and Statistics (AISTATS22). Proceedings of Machine Learning Research, vol. 151, pp. 4574\u20134594. PMLR (2022)"},{"key":"16_CR37","unstructured":"Pawelczyk, M., Broelemann, K., Kasneci, G.: On counterfactual explanations under predictive multiplicity. In: Proceedings of the 36th Conference on Uncertainty in Artificial Intelligence (UAI20). Proceedings of Machine Learning Research, vol. 124, pp. 809\u2013818. AUAI Press (2020)"},{"key":"16_CR38","unstructured":"Pawelczyk, M., Datta, T., van den Heuvel, J., Kasneci, G., Lakkaraju, H.: Probabilistically robust recourse: navigating the trade-offs between costs and robustness in algorithmic recourse. In: Proceedings of the 11th International Conference on Learning Representations (ICLR23). OpenReview.net (2023)"},{"key":"16_CR39","unstructured":"ProPublica: How We Analyzed the COMPAS Recidivism Algorithm (2016). https:\/\/www.propublica.org\/article\/how-we-analyzed-the-compas-recidivism-algorithm"},{"key":"16_CR40","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"243","DOI":"10.1007\/978-3-642-14295-6_24","volume-title":"Computer Aided Verification","author":"L Pulina","year":"2010","unstructured":"Pulina, L., Tacchella, A.: An abstraction-refinement approach to verification of artificial neural networks. In: Touili, T., Cook, B., Jackson, P. (eds.) CAV 2010. LNCS, vol. 6174, pp. 243\u2013257. Springer, Heidelberg (2010). https:\/\/doi.org\/10.1007\/978-3-642-14295-6_24"},{"issue":"6","key":"16_CR41","doi-asserted-by":"publisher","first-page":"673","DOI":"10.1007\/s10458-019-09408-y","volume":"33","author":"A Rosenfeld","year":"2019","unstructured":"Rosenfeld, A., Richardson, A.: Explainability in human-agent systems. Auton. Agents Multi Agent Syst. 33(6), 673\u2013705 (2019)","journal-title":"Auton. Agents Multi Agent Syst."},{"key":"16_CR42","doi-asserted-by":"crossref","unstructured":"Russell, C.: Efficient search for diverse coherent explanations. In: Proceedings of the International Conference on Fairness, Accountability, and Transparency (FAT*19), pp. 20\u201328. ACM (2019)","DOI":"10.1145\/3287560.3287569"},{"key":"16_CR43","doi-asserted-by":"crossref","unstructured":"Sharma, S., Henderson, J., Ghosh, J.: CERTIFAI: a common framework to provide explanations and analyse the fairness and robustness of black-box models. In: Proceedings of the AAAI\/ACM Conference on AI, Ethics, and Society (AIES20), pp. 166\u2013172. ACM (2020)","DOI":"10.1145\/3375627.3375812"},{"key":"16_CR44","unstructured":"Slack, D., Hilgard, A., Lakkaraju, H., Singh, S.: Counterfactual explanations can be manipulated. In: Advances in Neural Information Processing Systems 34 (NeurIPS21), pp. 62\u201375 (2021)"},{"key":"16_CR45","doi-asserted-by":"crossref","unstructured":"Stallkamp, J., Schlipsing, M., Salmen, J., Igel, C.: The German traffic sign recognition benchmark: a multi-class classification competition. In: Proceedings of the International Joint Conference on Neural Networks (IJCNN11), pp. 1453\u20131460. IEEE (2011)","DOI":"10.1109\/IJCNN.2011.6033395"},{"key":"16_CR46","unstructured":"Upadhyay, S., Joshi, S., Lakkaraju, H.: Towards robust and reliable algorithmic recourse. In: Advances in Neural Information Processing Systems 34 (NeurIPS21), pp. 16926\u201316937 (2021)"},{"key":"16_CR47","doi-asserted-by":"crossref","unstructured":"Ustun, B., Spangher, A., Liu, Y.: Actionable recourse in linear classification. In: Proceedings of the Conference on Fairness, Accountability, and Transparency (FAT*19), pp. 10\u201319. ACM (2019)","DOI":"10.1145\/3287560.3287566"},{"key":"16_CR48","first-page":"841","volume":"31","author":"S Wachter","year":"2017","unstructured":"Wachter, S., Mittelstadt, B., Russell, C.: Counterfactual explanations without opening the black box: automated decisions and the GDPR. Harv. JL Tech. 31, 841 (2017)","journal-title":"Harv. JL Tech."},{"key":"16_CR49","unstructured":"Wang, S., Pei, K., Whitehouse, J., Yang, J., Jana, S.: Efficient formal safety analysis of neural networks. In: Advances in Neural Information Processing Systems (NeurIPS18), pp. 6367\u20136377. Curran Associates, Inc. (2018)"}],"container-title":["Lecture Notes in Computer Science","Multi-Agent Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-43264-4_16","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,27]],"date-time":"2024-10-27T16:21:19Z","timestamp":1730046079000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-43264-4_16"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031432637","9783031432644"],"references-count":49,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-43264-4_16","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"7 September 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"EUMAS","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Multi-Agent Systems","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Naples","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Italy","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14 September 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"15 September 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"20","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"eumas2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/vadimmalvone.github.io\/eumas2023\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"47","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"24","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"5","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"51% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"PhD - Short Papers (20 submissions; 16 short papers accepted for proceedings)","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}